new changes (#3651)

Author: bn023052

tests/e2e/bootstrap/bootstrap.go

@@ -26,6 +26,7 @@ import (
 	cnstypes "github.com/vmware/govmomi/cns/types"
 	"k8s.io/kubernetes/test/e2e/framework"
 	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/clients/cns"
 	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/clients/vc"
 	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/config"
 	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants"
@@ -53,6 +54,8 @@ func Bootstrap(optionArgs ...bool) *config.E2eTestConfig {
 		e2eTestConfig.TestInput.Global.VCenterPort, e2eTestConfig.TestInput.Global.User,
 		e2eTestConfig.TestInput.Global.Password)
 
+	e2eTestConfig.CnsClient, _ = cns.NewCnsClient(ctx, e2eTestConfig.VcClient.Client)
+
 	if framework.TestContext.RepoRoot != "" {
 		testfiles.AddFileSource(testfiles.RootFileSource{Root: framework.TestContext.RepoRoot})
 	}

tests/e2e/config/test_config.go

@@ -134,6 +134,7 @@ type TestBedConfig struct {
 	Multivc             bool
 	StretchedSVC        bool
 	IsPrivateNetwork    bool
+	LinkedClone         bool
 	K8sMasterIp1PortNum string
 	K8sMasterIp2PortNum string
 	K8sMasterIp3PortNum string

tests/e2e/constants/env_constants.go

@@ -225,3 +225,9 @@ const (
 	EnvIsDevopsUser     = "IS_DEVOPS_USER"
 	EnvDevopsUserName   = "DEVOPS_USERNAME"
 )
+
+// For linked clone
+var (
+	EnvStoragePolicy          = "STORAGE_POLICY_NAME-LC"
+	EnvTargetDSUrlForRelocate = "TARGET_DS_URL-LC"
+)

tests/e2e/constants/kubernetes.go

@@ -138,6 +138,7 @@ const (
 	StorageQuotaWebhookPrefix  = "storage-quota-webhook"
 	DevopsKubeConf             = "DEV_OPS_USER_KUBECONFIG"
 	QuotaSupportedVCVersion    = "9.0.0"
+	LinkedCloneAnnotationKey   = "csi.vsphere.volume/fast-provisioning"
 )
 
 // For busybox pod image

tests/e2e/constants/test_suite_level.go

@@ -87,4 +87,5 @@ const (
 	VcptocsiTest          = "vcptocsiTest"
 	StretchedSvc          = "stretchedSvc"
 	Devops                = "devops"
+	LinkedClone           = "lc"
 )

tests/e2e/constants/vsphere_constants.go

@@ -23,6 +23,7 @@ const (
 	DiskSize1GB             = "1Gi"
 	DiskSize                = "2Gi"
 	DiskSizeLarge           = "100Gi"
+	DiskSize10GB            = "10Gi"
 	DiskSizeInMb            = int64(2048)
 	DiskSizeInMinMb         = int64(200)
 	E2eTestPassword         = "E2E-test-password!23"

tests/e2e/csisnapshot/util.go

@@ -27,15 +27,21 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
 	v1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/kubernetes/test/e2e/framework"
+	e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
+	fpv "k8s.io/kubernetes/test/e2e/framework/pv"
 	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/config"
 	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/constants"
 	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/env"
+	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/k8testutil"
 	"sigs.k8s.io/vsphere-csi-driver/v3/tests/e2e/vcutil"
 )
 
@@ -389,3 +395,283 @@ func WaitForVolumeSnapshotContentToBeDeleted(client snapclient.Clientset, ctx co
 		})
 	return waitErr
 }
+
+// GetRestConfigClientForGuestCluster can be used to get the KUBECONFIG
+func GetRestConfigClientForGuestCluster(guestClusterRestConfig *rest.Config) *rest.Config {
+	var err error
+	if guestClusterRestConfig == nil {
+		if k8senv := env.GetAndExpectStringEnvVar("KUBECONFIG"); k8senv != "" {
+			guestClusterRestConfig, err = clientcmd.BuildConfigFromFlags("", k8senv)
+		}
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	}
+	return guestClusterRestConfig
+}
+
+// verifyVolumeRestoreOperation verifies if volume(PVC) restore from given snapshot
+// and creates pod and checks attach volume operation if verifyPodCreation is set to true
+func VerifyVolumeRestoreOperation(ctx context.Context, e2eTestConfig *config.E2eTestConfig, client clientset.Interface,
+	namespace string, storageclass *storagev1.StorageClass,
+	volumeSnapshot *snapV1.VolumeSnapshot, diskSize string,
+	verifyPodCreation bool) (*v1.PersistentVolumeClaim, []*v1.PersistentVolume, *v1.Pod) {
+
+	ginkgo.By("Create PVC from snapshot")
+	pvcSpec := GetPersistentVolumeClaimSpecWithDatasource(namespace, diskSize, storageclass, nil,
+		v1.ReadWriteOnce, volumeSnapshot.Name, constants.Snapshotapigroup)
+
+	pvclaim2, err := fpv.CreatePVC(ctx, client, namespace, pvcSpec)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	persistentvolumes2, err := fpv.WaitForPVClaimBoundPhase(ctx, client,
+		[]*v1.PersistentVolumeClaim{pvclaim2}, framework.ClaimProvisionTimeout)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	volHandle2 := persistentvolumes2[0].Spec.CSI.VolumeHandle
+	if e2eTestConfig.TestInput.ClusterFlavor.GuestCluster {
+		volHandle2 = k8testutil.GetVolumeIDFromSupervisorCluster(volHandle2)
+	}
+	gomega.Expect(volHandle2).NotTo(gomega.BeEmpty())
+
+	var pod *v1.Pod
+	if verifyPodCreation {
+		// Create a Pod to use this PVC, and verify volume has been attached
+		ginkgo.By("Creating pod to attach PV to the node")
+		pod, err = k8testutil.CreatePod(ctx, e2eTestConfig, client, namespace, nil,
+			[]*v1.PersistentVolumeClaim{pvclaim2}, false, constants.ExecRWXCommandPod1)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		var vmUUID string
+		var exists bool
+		nodeName := pod.Spec.NodeName
+
+		if e2eTestConfig.TestInput.ClusterFlavor.VanillaCluster {
+			vmUUID = k8testutil.GetNodeUUID(ctx, client, pod.Spec.NodeName)
+		} else if e2eTestConfig.TestInput.ClusterFlavor.GuestCluster {
+			vmUUID, err = vcutil.GetVMUUIDFromNodeName(e2eTestConfig, pod.Spec.NodeName)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		} else if e2eTestConfig.TestInput.ClusterFlavor.SupervisorCluster {
+			annotations := pod.Annotations
+			vmUUID, exists = annotations[constants.VmUUIDLabel]
+			gomega.Expect(exists).To(gomega.BeTrue(), fmt.Sprintf("Pod doesn't have %s annotation", constants.VmUUIDLabel))
+			_, err := vcutil.GetVMByUUID(ctx, e2eTestConfig, vmUUID)
+			gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		}
+
+		ginkgo.By(fmt.Sprintf("Verify volume: %s is attached to the node: %s", volHandle2, nodeName))
+		isDiskAttached, err := vcutil.IsVolumeAttachedToVM(client, e2eTestConfig, volHandle2, vmUUID)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+		gomega.Expect(isDiskAttached).To(gomega.BeTrue(), "Volume is not attached to the node")
+
+		ginkgo.By("Verify the volume is accessible and Read/write is possible")
+		var cmd []string
+		if e2eTestConfig.TestInput.TestBedInfo.WindowsEnv {
+			cmd = []string{"exec", pod.Name, "--namespace=" + namespace, "powershell.exe", "cat ", constants.FilePathPod1}
+		} else {
+			cmd = []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c",
+				"cat ", constants.FilePathPod1}
+		}
+		output := e2ekubectl.RunKubectlOrDie(namespace, cmd...)
+		gomega.Expect(strings.Contains(output, "Hello message from Pod1")).NotTo(gomega.BeFalse())
+
+		var wrtiecmd []string
+		if e2eTestConfig.TestInput.TestBedInfo.WindowsEnv {
+			wrtiecmd = []string{"exec", pod.Name, "--namespace=" + namespace, "powershell.exe",
+				"Add-Content /mnt/volume1/Pod1.html 'Hello message from test into Pod1'"}
+		} else {
+			wrtiecmd = []string{"exec", pod.Name, "--namespace=" + namespace, "--", "/bin/sh", "-c",
+				"echo 'Hello message from test into Pod1' >> /mnt/volume1/Pod1.html"}
+		}
+		e2ekubectl.RunKubectlOrDie(namespace, wrtiecmd...)
+		output = e2ekubectl.RunKubectlOrDie(namespace, cmd...)
+		gomega.Expect(strings.Contains(output, "Hello message from test into Pod1")).NotTo(gomega.BeFalse())
+		return pvclaim2, persistentvolumes2, pod
+	}
+	return pvclaim2, persistentvolumes2, pod
+}
+
+/* createPreProvisionedSnapshotInGuestCluster created pre-provisioned snaphot  on Guest cluster */
+func CreatePreProvisionedSnapshotInGuestCluster(ctx context.Context, volumeSnapshot *snapV1.VolumeSnapshot,
+	updatedSnapshotContent *snapV1.VolumeSnapshotContent,
+	snapc *snapclient.Clientset, namespace string, pandoraSyncWaitTime int,
+	svcVolumeSnapshotName string, diskSize string) (*snapV1.VolumeSnapshotContent,
+	*snapV1.VolumeSnapshot, bool, bool, error) {
+
+	framework.Logf("Change the deletion policy of VolumeSnapshotContent from Delete to Retain " +
+		"in Guest Cluster")
+
+	updatedSnapshotContent, err := ChangeDeletionPolicyOfVolumeSnapshotContent(ctx, updatedSnapshotContent,
+		snapc, snapV1.VolumeSnapshotContentRetain)
+	if err != nil {
+		return nil, nil, false, false, fmt.Errorf("failed to change deletion policy of VolumeSnapshotContent: %v", err)
+	}
+
+	framework.Logf("Delete dynamic volume snapshot from Guest Cluster")
+	DeleteVolumeSnapshotWithPandoraWait(ctx, snapc, namespace, volumeSnapshot.Name, pandoraSyncWaitTime)
+
+	framework.Logf("Delete VolumeSnapshotContent from Guest Cluster explicitly")
+	err = DeleteVolumeSnapshotContent(ctx, updatedSnapshotContent, snapc, pandoraSyncWaitTime)
+	if err != nil {
+		return nil, nil, false, false, fmt.Errorf("failed to delete VolumeSnapshotContent: %v", err)
+	}
+
+	framework.Logf("Creating static VolumeSnapshotContent in Guest Cluster using "+
+		"supervisor VolumeSnapshotName %s", svcVolumeSnapshotName)
+	staticSnapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Create(ctx,
+		GetVolumeSnapshotContentSpec(snapV1.DeletionPolicy("Delete"), svcVolumeSnapshotName,
+			"static-vs", namespace), metav1.CreateOptions{})
+	if err != nil {
+		return nil, nil, false, false, fmt.Errorf("failed to create static VolumeSnapshotContent: %v", err)
+	}
+
+	framework.Logf("Verify VolumeSnapshotContent is created or not in Guest Cluster")
+	staticSnapshotContent, err = snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx,
+		staticSnapshotContent.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, nil, false, false, fmt.Errorf("failed to get static VolumeSnapshotContent: %v", err)
+	}
+	framework.Logf("Snapshotcontent name is  %s", staticSnapshotContent.ObjectMeta.Name)
+
+	staticSnapshotContent, err = WaitForVolumeSnapshotContentReadyToUse(*snapc, ctx, staticSnapshotContent.Name)
+	if err != nil {
+		return nil, nil, false, false, fmt.Errorf("volume snapshot content is not ready to use")
+	}
+	staticSnapshotContentCreated := true
+
+	ginkgo.By("Create a static volume snapshot by static snapshotcontent")
+	staticVolumeSnapshot, err := snapc.SnapshotV1().VolumeSnapshots(namespace).Create(ctx,
+		GetVolumeSnapshotSpecByName(namespace, "static-vs",
+			staticSnapshotContent.ObjectMeta.Name), metav1.CreateOptions{})
+	if err != nil {
+		return nil, nil, false, false, fmt.Errorf("failed to create static volume snapshot: %v", err)
+	}
+	framework.Logf("Volume snapshot name is : %s", staticVolumeSnapshot.Name)
+
+	ginkgo.By("Verify static volume snapshot is created")
+	staticSnapshot, err := WaitForVolumeSnapshotReadyToUse(*snapc, ctx, namespace, staticVolumeSnapshot.Name)
+	if err != nil {
+		return nil, nil, false, false, fmt.Errorf("volumeSnapshot is still not ready to use: %v", err)
+	}
+	if staticSnapshot.Status.RestoreSize.Cmp(resource.MustParse(diskSize)) != 0 {
+		return nil, nil, false, false, fmt.Errorf("expected RestoreSize does not match")
+	}
+	framework.Logf("Snapshot details is %+v", staticSnapshot)
+	staticSnapshotCreated := true
+
+	return staticSnapshotContent, staticSnapshot, staticSnapshotContentCreated, staticSnapshotCreated, nil
+}
+
+/*
+changeDeletionPolicyOfVolumeSnapshotContentOnGuest changes the deletion policy
+of volume snapshot content from delete to retain in Guest Cluster
+*/
+func ChangeDeletionPolicyOfVolumeSnapshotContent(ctx context.Context,
+	snapshotContent *snapV1.VolumeSnapshotContent, snapc *snapclient.Clientset,
+	policyName snapV1.DeletionPolicy) (*snapV1.VolumeSnapshotContent, error) {
+
+	// Retrieve the latest version of the object
+	latestSnapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Get(ctx, snapshotContent.Name,
+		metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	// Apply changes to the latest version
+	latestSnapshotContent.Spec.DeletionPolicy = policyName
+
+	// Update the object
+	updatedSnapshotContent, err := snapc.SnapshotV1().VolumeSnapshotContents().Update(ctx,
+		latestSnapshotContent, metav1.UpdateOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	return updatedSnapshotContent, nil
+}
+
+// getVolumeSnapshotContentSpec returns a spec for the volume snapshot content
+func GetVolumeSnapshotContentSpec(deletionPolicy snapV1.DeletionPolicy, snapshotHandle string,
+	futureSnapshotName string, namespace string) *snapV1.VolumeSnapshotContent {
+	var volumesnapshotContentSpec = &snapV1.VolumeSnapshotContent{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "VolumeSnapshotContent",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "snapshotcontent-",
+		},
+		Spec: snapV1.VolumeSnapshotContentSpec{
+			DeletionPolicy: deletionPolicy,
+			Driver:         constants.E2evSphereCSIDriverName,
+			Source: snapV1.VolumeSnapshotContentSource{
+				SnapshotHandle: &snapshotHandle,
+			},
+			VolumeSnapshotRef: v1.ObjectReference{
+				Name:      futureSnapshotName,
+				Namespace: namespace,
+			},
+		},
+	}
+	return volumesnapshotContentSpec
+}
+
+// getVolumeSnapshotSpecByName returns a spec for the volume snapshot by name
+func GetVolumeSnapshotSpecByName(namespace string, snapshotName string,
+	snapshotcontentname string) *snapV1.VolumeSnapshot {
+	var volumesnapshotSpec = &snapV1.VolumeSnapshot{
+		TypeMeta: metav1.TypeMeta{
+			Kind: "VolumeSnapshot",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      snapshotName,
+			Namespace: namespace,
+		},
+		Spec: snapV1.VolumeSnapshotSpec{
+			Source: snapV1.VolumeSnapshotSource{
+				VolumeSnapshotContentName: &snapshotcontentname,
+			},
+		},
+	}
+	return volumesnapshotSpec
+}
+
+// GetPersistentVolumeClaimSpecWithDatasource return the PersistentVolumeClaim
+// spec with specified storage class.
+func GetPersistentVolumeClaimSpecWithDatasource(namespace string, ds string, storageclass *storagev1.StorageClass,
+	pvclaimlabels map[string]string, accessMode v1.PersistentVolumeAccessMode,
+	datasourceName string, snapshotapigroup string) *v1.PersistentVolumeClaim {
+	disksize := constants.DiskSize
+	if ds != "" {
+		disksize = ds
+	}
+	if accessMode == "" {
+		// If accessMode is not specified, set the default accessMode.
+		accessMode = v1.ReadWriteOnce
+	}
+	claim := &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-",
+			Namespace:    namespace,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				accessMode,
+			},
+			Resources: v1.VolumeResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse(disksize),
+				},
+			},
+			StorageClassName: &(storageclass.Name),
+			DataSource: &v1.TypedLocalObjectReference{
+				APIGroup: &snapshotapigroup,
+				Kind:     "VolumeSnapshot",
+				Name:     datasourceName,
+			},
+		},
+	}
+
+	if pvclaimlabels != nil {
+		claim.Labels = pvclaimlabels
+	}
+
+	return claim
+}