Add volume modificatoin recovery e2e test

Author: AndrewSirenko

test/e2e/framework/pv/wait.go

@@ -68,3 +68,42 @@ func WaitForPersistentVolumeClaimModified(ctx context.Context, c clientset.Inter
 			}, nil
 		}))
 }
+
+// WaitForPersistentVolumeClaimModificationFailure waits the given timeout duration for the specified claim to have
+// failed to bind to the invalid volume attributes class.
+// Returns an error if timeout occurs first.
+func WaitForPersistentVolumeClaimModificationFailure(ctx context.Context, c clientset.Interface, claim *v1.PersistentVolumeClaim, timeout time.Duration) error {
+	desiredClass := ptr.Deref(claim.Spec.VolumeAttributesClassName, "")
+
+	var match = func(claim *v1.PersistentVolumeClaim) bool {
+		for _, condition := range claim.Status.Conditions {
+			if condition.Type != v1.PersistentVolumeClaimVolumeModifyVolumeError {
+				return false
+			}
+		}
+
+		// check if claim's current volume attributes class is NOT desired one, and has appropriate ModifyVolumeStatus
+		currentClass := ptr.Deref(claim.Status.CurrentVolumeAttributesClassName, "")
+		return claim.Status.Phase == v1.ClaimBound &&
+			desiredClass != currentClass && claim.Status.ModifyVolumeStatus != nil &&
+			(claim.Status.ModifyVolumeStatus.Status == v1.PersistentVolumeClaimModifyVolumeInProgress ||
+				claim.Status.ModifyVolumeStatus.Status == v1.PersistentVolumeClaimModifyVolumeInfeasible)
+	}
+
+	if match(claim) {
+		return nil
+	}
+
+	return framework.Gomega().
+		Eventually(ctx, framework.GetObject(c.CoreV1().PersistentVolumeClaims(claim.Namespace).Get, claim.Name, metav1.GetOptions{})).
+		WithTimeout(timeout).
+		Should(framework.MakeMatcher(func(claim *v1.PersistentVolumeClaim) (func() string, error) {
+			if match(claim) {
+				return nil, nil
+			}
+
+			return func() string {
+				return fmt.Sprintf("expected claim's status to NOT be modified with the given VolumeAttirbutesClass %s, got instead:\n%s", desiredClass, format.Object(claim, 1))
+			}, nil
+		}))
+}

test/e2e/storage/testsuites/volume_modify.go

@@ -41,11 +41,10 @@ import (
 )
 
 const (
-	modifyPollInterval               = 2 * time.Second
-	setVACWaitPeriod                 = 30 * time.Second
-	modifyingConditionSyncWaitPeriod = 2 * time.Minute
-	modifyVolumeWaitPeriod           = 10 * time.Minute
-	vacCleanupWaitPeriod             = 30 * time.Second
+	modifyPollInterval     = 2 * time.Second
+	setVACWaitPeriod       = 30 * time.Second
+	modifyVolumeWaitPeriod = 10 * time.Minute
+	vacCleanupWaitPeriod   = 30 * time.Second
 )
 
 type volumeModifyTestSuite struct {
@@ -227,6 +226,46 @@ func (v *volumeModifyTestSuite) DefineTests(driver storageframework.TestDriver,
 		err = e2epv.WaitForPersistentVolumeClaimModified(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod)
 		framework.ExpectNoError(err, "While waiting for PVC to have expected VAC")
 	})
+
+	ginkgo.It("should recover from invalid target VAC by updating PVC to new valid VAC", func(ctx context.Context) {
+		init(ctx, false /* volume created without VAC */)
+		ginkgo.DeferCleanup(cleanup)
+
+		// Create VAC with unsupported parameter
+		invalidVAC := MakeInvalidVAC(l.config)
+		_, err := f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Create(ctx, invalidVAC, metav1.CreateOptions{})
+		framework.ExpectNoError(err, "While creating new VolumeAttributesClass")
+		ginkgo.DeferCleanup(CleanupVAC, invalidVAC, f.ClientSet, vacCleanupWaitPeriod)
+
+		ginkgo.By("Creating a pod with dynamically provisioned volume")
+		podConfig := e2epod.Config{
+			NS:            f.Namespace.Name,
+			PVCs:          []*v1.PersistentVolumeClaim{l.resource.Pvc},
+			SeLinuxLabel:  e2epod.GetLinuxLabel(),
+			NodeSelection: l.config.ClientNodeSelection,
+			ImageID:       e2epod.GetDefaultTestImageID(),
+		}
+		pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
+		ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
+		framework.ExpectNoError(err, "While creating pod for modifying")
+
+		ginkgo.By("Attempting to modify PVC via VolumeAttributeClass that contains unsupported parameters")
+		newPVC := SetPVCVACName(ctx, l.resource.Pvc, invalidVAC.Name, f.ClientSet, setVACWaitPeriod)
+		l.resource.Pvc = newPVC
+		gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil())
+
+		ginkgo.By("Waiting for modification to fail")
+		err = e2epv.WaitForPersistentVolumeClaimModificationFailure(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod)
+		framework.ExpectNoError(err, "While waiting for PVC to have conditions")
+
+		ginkgo.By("Modifying PVC to new valid VAC")
+		l.resource.Pvc = SetPVCVACName(ctx, l.resource.Pvc, l.vac.Name, f.ClientSet, setVACWaitPeriod)
+		gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil())
+
+		ginkgo.By("Checking PVC status")
+		err = e2epv.WaitForPersistentVolumeClaimModified(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod)
+		framework.ExpectNoError(err, "While waiting for PVC to have expected VAC")
+	})
 }
 
 // SetPVCVACName sets the VolumeAttributesClassName on a PVC object
@@ -246,6 +285,15 @@ func SetPVCVACName(ctx context.Context, origPVC *v1.PersistentVolumeClaim, name
 	return patchedPVC
 }
 
+func MakeInvalidVAC(config *storageframework.PerTestConfig) *storagev1beta1.VolumeAttributesClass {
+	return storageframework.CopyVolumeAttributesClass(&storagev1beta1.VolumeAttributesClass{
+		DriverName: config.GetUniqueDriverName(),
+		Parameters: map[string]string{
+			"xxInvalidParameterKey": "xxInvalidParameterValue",
+		},
+	}, config.Framework.Namespace.Name, "e2e-vac-invalid")
+}
+
 func CleanupVAC(ctx context.Context, vac *storagev1beta1.VolumeAttributesClass, c clientset.Interface, timeout time.Duration) {
 	gomega.Eventually(ctx, func() error {
 		return c.StorageV1beta1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})