Merge pull request kubernetes#128972 from manishym/beta-e2e

Add crd from external snapshotter

Author: k8s-ci-robot

test/e2e/storage/framework/volume_group_snapshot_resource.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 
 	"github.com/onsi/ginkgo/v2"
-
+	"github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -56,15 +56,15 @@ type VolumeGroupSnapshotResource struct {
 	Config  *PerTestConfig
 	Pattern TestPattern
 
-	Vgs        *unstructured.Unstructured
-	Vgscontent *unstructured.Unstructured
-	Vgsclass   *unstructured.Unstructured
+	VGS        *unstructured.Unstructured
+	VGSContent *unstructured.Unstructured
+	VGSClass   *unstructured.Unstructured
 }
 
 // CreateVolumeGroupSnapshot creates a VolumeGroupSnapshotClass with given SnapshotDeletionPolicy and a VolumeGroupSnapshot
 // from the VolumeGroupSnapshotClass using a dynamic client.
 // Returns the unstructured VolumeGroupSnapshotClass and VolumeGroupSnapshot objects.
-func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured) {
+func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured, *unstructured.Unstructured) {
 	defer ginkgo.GinkgoRecover()
 	var err error
 	if pattern.SnapshotType != VolumeGroupSnapshot {
@@ -99,28 +99,35 @@ func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshott
 	ginkgo.By("Getting group snapshot and content")
 	volumeGroupSnapshot, err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(volumeGroupSnapshot.GetNamespace()).Get(ctx, volumeGroupSnapshot.GetName(), metav1.GetOptions{})
 	framework.ExpectNoError(err, "Failed to get volume group snapshot after creation")
-
-	return gsclass, volumeGroupSnapshot
+	status := volumeGroupSnapshot.Object["status"]
+	err = framework.Gomega().Expect(status).NotTo(gomega.BeNil())
+	framework.ExpectNoError(err, "Failed to get status of volume group snapshot")
+	vgscName := status.(map[string]interface{})["boundVolumeGroupSnapshotContentName"].(string)
+	err = framework.Gomega().Expect(vgscName).NotTo(gomega.BeNil())
+	framework.ExpectNoError(err, "Failed to get content name of volume group snapshot")
+	vgsc, err := dc.Resource(utils.VolumeGroupSnapshotContentGVR).Get(ctx, vgscName, metav1.GetOptions{})
+	framework.ExpectNoError(err, "failed to get content of group snapshot")
+	return gsclass, volumeGroupSnapshot, vgsc
 }
 
 // CleanupResource deletes the VolumeGroupSnapshotClass and VolumeGroupSnapshot objects using a dynamic client.
 func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error {
 	defer ginkgo.GinkgoRecover()
 	dc := r.Config.Framework.DynamicClient
-	err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.Vgsclass.GetName(), metav1.DeleteOptions{})
+	err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.VGSClass.GetName(), metav1.DeleteOptions{})
 	framework.ExpectNoError(err, "Failed to delete volume group snapshot class")
 	return nil
 }
 
 // CreateVolumeGroupSnapshotResource creates a VolumeGroupSnapshotResource object with the given parameters.
 func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource {
-	vgsclass, snapshot := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
+	vgsClass, snapshot, vgsc := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
 	vgs := &VolumeGroupSnapshotResource{
 		Config:     config,
 		Pattern:    pattern,
-		Vgs:        snapshot,
-		Vgsclass:   vgsclass,
-		Vgscontent: nil,
+		VGS:        snapshot,
+		VGSClass:   vgsClass,
+		VGSContent: vgsc,
 	}
 	return vgs
 }

test/e2e/storage/testsuites/volume_group_snapshottable.go

@@ -18,6 +18,8 @@ package testsuites
 
 import (
 	"context"
+	"crypto/sha256"
+	"fmt"
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
@@ -176,18 +178,30 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
 				snapshot := storageframework.CreateVolumeGroupSnapshotResource(ctx, snapshottableDriver, groupTest.config, pattern, labelValue, groupTest.volumeGroup[0][0].Pvc.GetNamespace(), f.Timeouts, map[string]string{"deletionPolicy": pattern.SnapshotDeletionPolicy.String()})
 				groupTest.snapshots = append(groupTest.snapshots, snapshot)
 				ginkgo.By("verifying the snapshots in the group are ready to use")
-				status := snapshot.Vgs.Object["status"]
+				status := snapshot.VGS.Object["status"]
 				err := framework.Gomega().Expect(status).NotTo(gomega.BeNil())
 				framework.ExpectNoError(err, "failed to get status of group snapshot")
-				volumes := status.(map[string]interface{})["pvcVolumeSnapshotRefList"]
-				err = framework.Gomega().Expect(volumes).NotTo(gomega.BeNil())
+
+				volumeListMap := snapshot.VGSContent.Object["status"].(map[string]interface{})
+				err = framework.Gomega().Expect(volumeListMap).NotTo(gomega.BeNil())
+				framework.ExpectNoError(err, "failed to get volume snapshot list")
+				volumeSnapshotHandlePairList := volumeListMap["volumeSnapshotHandlePairList"].([]interface{})
+				err = framework.Gomega().Expect(volumeSnapshotHandlePairList).NotTo(gomega.BeNil())
 				framework.ExpectNoError(err, "failed to get volume snapshot list")
-				volumeList := volumes.([]interface{})
-				err = framework.Gomega().Expect(len(volumeList)).To(gomega.Equal(groupTest.numVolumes))
+				err = framework.Gomega().Expect(len(volumeSnapshotHandlePairList)).To(gomega.Equal(groupTest.numVolumes))
 				framework.ExpectNoError(err, "failed to get volume snapshot list")
 				claimSize := groupTest.volumeGroup[0][0].Pvc.Spec.Resources.Requests.Storage().String()
-				for _, volume := range volumeList {
+				for _, volume := range volumeSnapshotHandlePairList {
 					// Create a PVC from the snapshot
+					volumeHandle := volume.(map[string]interface{})["volumeHandle"].(string)
+					err = framework.Gomega().Expect(volumeHandle).NotTo(gomega.BeNil())
+					framework.ExpectNoError(err, "failed to get volume handle from volume")
+					uid := snapshot.VGSContent.Object["metadata"].(map[string]interface{})["uid"].(string)
+					err = framework.Gomega().Expect(uid).NotTo(gomega.BeNil())
+					framework.ExpectNoError(err, "failed to get uuid from content")
+					volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
+						uid+volumeHandle)))
+
 					pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
 						StorageClassName: &groupTest.volumeGroup[0][0].Sc.Name,
 						ClaimSize:        claimSize,
@@ -198,7 +212,7 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
 					pvc.Spec.DataSource = &v1.TypedLocalObjectReference{
 						APIGroup: &group,
 						Kind:     "VolumeSnapshot",
-						Name:     volume.(map[string]interface{})["volumeSnapshotRef"].(map[string]interface{})["name"].(string),
+						Name:     volumeSnapshotName,
 					}
 
 					volSrc := v1.VolumeSource{
@@ -208,8 +222,9 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
 							},
 						},
 					}
-					pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
+					pvc, err = cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
 					framework.ExpectNoError(err, "failed to create PVC from snapshot")
+
 					pod := StartInPodWithVolumeSource(ctx, cs, volSrc, pvc.Namespace, "snapshot-pod", "sleep 300", groupTest.config.ClientNodeSelection)
 					ginkgo.DeferCleanup(e2epod.DeletePodWithWait, cs, pod)
 					framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow), "Pod did not start in expected time")

test/e2e/storage/utils/volume_group_snapshot.go

@@ -33,15 +33,16 @@ const (
 	// VolumeGroupSnapshot is the group snapshot api
 	VolumeGroupSnapshotAPIGroup = "groupsnapshot.storage.k8s.io"
 	// VolumeGroupSnapshotAPIVersion is the group snapshot api version
-	VolumeGroupSnapshotAPIVersion = "groupsnapshot.storage.k8s.io/v1alpha1"
+	VolumeGroupSnapshotAPIVersion = "groupsnapshot.storage.k8s.io/v1beta1"
 )
 
 var (
 
 	// VolumeGroupSnapshotGVR is GroupVersionResource for volumegroupsnapshots
-	VolumeGroupSnapshotGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1alpha1", Resource: "volumegroupsnapshots"}
+	VolumeGroupSnapshotGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshots"}
 	// VolumeGroupSnapshotClassGVR is GroupVersionResource for volumegroupsnapshotsclasses
-	VolumeGroupSnapshotClassGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1alpha1", Resource: "volumegroupsnapshotclasses"}
+	VolumeGroupSnapshotClassGVR   = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshotclasses"}
+	VolumeGroupSnapshotContentGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshotcontents"}
 )
 
 // WaitForVolumeGroupSnapshotReady waits for a VolumeGroupSnapshot to be ready to use or until timeout occurs, whichever comes first.

test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814"
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
     controller-gen.kubebuilder.io/version: v0.15.0
   name: volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io
 spec:
@@ -31,7 +31,7 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
-    name: v1alpha1
+    name: v1beta1
     schema:
       openAPIV3Schema:
         description: |-

test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshotcontents.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1068"
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
     controller-gen.kubebuilder.io/version: v0.15.0
   name: volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io
 spec:
@@ -53,7 +53,7 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
-    name: v1alpha1
+    name: v1beta1
     schema:
       openAPIV3Schema:
         description: |-
@@ -237,8 +237,6 @@ spec:
                 - message: both volumeGroupSnapshotRef.name and volumeGroupSnapshotRef.namespace
                     must be set
                   rule: has(self.name) && has(self.__namespace__)
-                - message: volumeGroupSnapshotRef is immutable
-                  rule: self == oldSelf
             required:
             - deletionPolicy
             - driver
@@ -257,8 +255,9 @@ spec:
                   The format of this field is a Unix nanoseconds time encoded as an int64.
                   On Unix, the command date +%s%N returns the current time in nanoseconds
                   since 1970-01-01 00:00:00 UTC.
-                format: int64
-                type: integer
+                  This field is the source for the CreationTime field in VolumeGroupSnapshotStatus
+                format: date-time
+                type: string
               error:
                 description: |-
                   Error is the last observed error during group snapshot creation, if any.
@@ -276,42 +275,6 @@ spec:
                     format: date-time
                     type: string
                 type: object
-              pvVolumeSnapshotContentList:
-                description: |-
-                  PVVolumeSnapshotContentList is the list of pairs of PV and
-                  VolumeSnapshotContent for this group snapshot
-                  The maximum number of allowed snapshots in the group is 100.
-                items:
-                  description: |-
-                    PVVolumeSnapshotContentPair represent a pair of PV names and
-                    VolumeSnapshotContent names
-                  properties:
-                    persistentVolumeRef:
-                      description: PersistentVolumeRef is a reference to the persistent
-                        volume resource
-                      properties:
-                        name:
-                          description: |-
-                            Name of the referent.
-                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                            TODO: Add other useful fields. apiVersion, kind, uid?
-                          type: string
-                      type: object
-                      x-kubernetes-map-type: atomic
-                    volumeSnapshotContentRef:
-                      description: VolumeSnapshotContentRef is a reference to the
-                        volume snapshot content resource
-                      properties:
-                        name:
-                          description: |-
-                            Name of the referent.
-                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                            TODO: Add other useful fields. apiVersion, kind, uid?
-                          type: string
-                      type: object
-                      x-kubernetes-map-type: atomic
-                  type: object
-                type: array
               readyToUse:
                 description: |-
                   ReadyToUse indicates if all the individual snapshots in the group are ready to be
@@ -325,6 +288,32 @@ spec:
                   If a storage system does not provide such an id, the
                   CSI driver can choose to return the VolumeGroupSnapshot name.
                 type: string
+              volumeSnapshotHandlePairList:
+                description: |-
+                  VolumeSnapshotHandlePairList is a list of CSI "volume_id" and "snapshot_id"
+                  pair returned by the CSI driver to identify snapshots and their source volumes
+                  on the storage system.
+                items:
+                  description: VolumeSnapshotHandlePair defines a pair of a source
+                    volume handle and a snapshot handle
+                  properties:
+                    snapshotHandle:
+                      description: |-
+                        SnapshotHandle is a unique id returned by the CSI driver to identify a volume
+                        snapshot on the storage system
+                        Required.
+                      type: string
+                    volumeHandle:
+                      description: |-
+                        VolumeHandle is a unique id returned by the CSI driver to identify a volume
+                        on the storage system
+                        Required.
+                      type: string
+                  required:
+                  - snapshotHandle
+                  - volumeHandle
+                  type: object
+                type: array
             type: object
         required:
         - spec

test/e2e/testing-manifests/storage-csi/external-snapshotter/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1068"
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
     controller-gen.kubebuilder.io/version: v0.15.0
   name: volumegroupsnapshots.groupsnapshot.storage.k8s.io
 spec:
@@ -43,7 +43,7 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
-    name: v1alpha1
+    name: v1beta1
     schema:
       openAPIV3Schema:
         description: |-
@@ -198,6 +198,7 @@ spec:
                   The format of this field is a Unix nanoseconds time encoded as an int64.
                   On Unix, the command date +%s%N returns the current time in nanoseconds
                   since 1970-01-01 00:00:00 UTC.
+                  This field is updated based on the CreationTime field in VolumeGroupSnapshotContentStatus
                 format: date-time
                 type: string
               error:
@@ -221,41 +222,6 @@ spec:
                     format: date-time
                     type: string
                 type: object
-              pvcVolumeSnapshotRefList:
-                description: |-
-                  VolumeSnapshotRefList is the list of PVC and VolumeSnapshot pairs that
-                  is part of this group snapshot.
-                  The maximum number of allowed snapshots in the group is 100.
-                items:
-                  description: PVCVolumeSnapshotPair defines a pair of a PVC reference
-                    and a Volume Snapshot Reference
-                  properties:
-                    persistentVolumeClaimRef:
-                      description: PersistentVolumeClaimRef is a reference to the
-                        PVC this pair is referring to
-                      properties:
-                        name:
-                          description: |-
-                            Name of the referent.
-                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                            TODO: Add other useful fields. apiVersion, kind, uid?
-                          type: string
-                      type: object
-                      x-kubernetes-map-type: atomic
-                    volumeSnapshotRef:
-                      description: VolumeSnapshotRef is a reference to the VolumeSnapshot
-                        this pair is referring to
-                      properties:
-                        name:
-                          description: |-
-                            Name of the referent.
-                            More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
-                            TODO: Add other useful fields. apiVersion, kind, uid?
-                          type: string
-                      type: object
-                      x-kubernetes-map-type: atomic
-                  type: object
-                type: array
               readyToUse:
                 description: |-
                   ReadyToUse indicates if all the individual snapshots in the group are ready

test/e2e/testing-manifests/storage-csi/external-snapshotter/volume-group-snapshots/csi-hostpath-plugin.yaml

@@ -219,7 +219,7 @@ spec:
       serviceAccountName: csi-hostpathplugin-sa
       containers:
         - name: hostpath
-          image: registry.k8s.io/sig-storage/hostpathplugin:v1.14.0
+          image: registry.k8s.io/sig-storage/hostpathplugin:v1.15.0
           args:
             - "--drivername=hostpath.csi.k8s.io"
             - "--v=5"
@@ -354,11 +354,11 @@ spec:
               name: socket-dir
 
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
           args:
             - -v=5
             - --csi-address=/csi/csi.sock
-            - --enable-volume-group-snapshots=true
+            - --feature-gates=CSIVolumeGroupSnapshot=true
           securityContext:
             # This is necessary only for systems with SELinux, where
             # non-privileged sidecar containers cannot access unix domain socket