Skip to content

Commit 0393497

Browse files
PhaowPhaow
committed
Add protection finalizer to vac when it is created
1 parent 18e5a4d commit 0393497

File tree

3 files changed

+228
-5
lines changed

3 files changed

+228
-5
lines changed

‎plugin/pkg/admission/storage/storageobjectinuseprotection/admission.go‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@ import (
2121
"io"
2222

2323
"k8s.io/apiserver/pkg/admission"
24+
"k8s.io/apiserver/pkg/util/feature"
2425
"k8s.io/klog/v2"
2526
api "k8s.io/kubernetes/pkg/apis/core"
27+
storageapi "k8s.io/kubernetes/pkg/apis/storage"
28+
"k8s.io/kubernetes/pkg/features"
2629
volumeutil "k8s.io/kubernetes/pkg/volume/util"
2730
)
2831

@@ -56,6 +59,7 @@ func newPlugin() *storageProtectionPlugin {
5659
var (
5760
pvResource = api.Resource("persistentvolumes")
5861
pvcResource = api.Resource("persistentvolumeclaims")
62+
vacResource = storageapi.Resource("volumeattributesclasses")
5963
)
6064

6165
// Admit sets finalizer on all PVCs(PVs). The finalizer is removed by
@@ -69,6 +73,11 @@ func (c *storageProtectionPlugin) Admit(ctx context.Context, a admission.Attribu
6973
return c.admitPV(a)
7074
case pvcResource:
7175
return c.admitPVC(a)
76+
case vacResource:
77+
if feature.DefaultFeatureGate.Enabled(features.VolumeAttributesClass) {
78+
return c.admitVAC(a)
79+
}
80+
return nil
7281

7382
default:
7483
return nil
@@ -119,3 +128,26 @@ func (c *storageProtectionPlugin) admitPVC(a admission.Attributes) error {
119128
pvc.Finalizers = append(pvc.Finalizers, volumeutil.PVCProtectionFinalizer)
120129
return nil
121130
}
131+
132+
func (c *storageProtectionPlugin) admitVAC(a admission.Attributes) error {
133+
if len(a.GetSubresource()) != 0 {
134+
return nil
135+
}
136+
137+
vac, ok := a.GetObject().(*storageapi.VolumeAttributesClass)
138+
// if we can't convert the obj to VAC, just return
139+
if !ok {
140+
klog.V(2).Infof("can't convert the obj to VAC to %s", vac.Name)
141+
return nil
142+
}
143+
for _, f := range vac.Finalizers {
144+
if f == volumeutil.VACProtectionFinalizer {
145+
// Finalizer is already present, nothing to do
146+
return nil
147+
}
148+
}
149+
klog.V(4).Infof("adding VAC protection finalizer to %s", vac.Name)
150+
vac.Finalizers = append(vac.Finalizers, volumeutil.VACProtectionFinalizer)
151+
152+
return nil
153+
}

‎plugin/pkg/admission/storage/storageobjectinuseprotection/admission_test.go‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,11 @@ import (
2626
"k8s.io/apimachinery/pkg/runtime/schema"
2727
"k8s.io/apimachinery/pkg/util/dump"
2828
"k8s.io/apiserver/pkg/admission"
29+
utilfeature "k8s.io/apiserver/pkg/util/feature"
30+
featuregatetesting "k8s.io/component-base/featuregate/testing"
2931
api "k8s.io/kubernetes/pkg/apis/core"
32+
storageapi "k8s.io/kubernetes/pkg/apis/storage"
33+
"k8s.io/kubernetes/pkg/features"
3034
volumeutil "k8s.io/kubernetes/pkg/volume/util"
3135
)
3236

@@ -49,12 +53,25 @@ func TestAdmit(t *testing.T) {
4953
Name: "pv",
5054
},
5155
}
56+
57+
vac := &storageapi.VolumeAttributesClass{
58+
TypeMeta: metav1.TypeMeta{
59+
Kind: "VolumeAttributesClass",
60+
},
61+
ObjectMeta: metav1.ObjectMeta{
62+
Name: "vac",
63+
},
64+
}
65+
5266
claimWithFinalizer := claim.DeepCopy()
5367
claimWithFinalizer.Finalizers = []string{volumeutil.PVCProtectionFinalizer}
5468

5569
pvWithFinalizer := pv.DeepCopy()
5670
pvWithFinalizer.Finalizers = []string{volumeutil.PVProtectionFinalizer}
5771

72+
vacWithFinalizer := vac.DeepCopy()
73+
vacWithFinalizer.Finalizers = []string{volumeutil.VACProtectionFinalizer}
74+
5875
tests := []struct {
5976
name string
6077
resource schema.GroupVersionResource
@@ -63,39 +80,58 @@ func TestAdmit(t *testing.T) {
6380
namespace string
6481
}{
6582
{
66-
"create -> add finalizer",
83+
"persistentvolumeclaims: create -> add finalizer",
6784
api.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
6885
claim,
6986
claimWithFinalizer,
7087
claim.Namespace,
7188
},
7289
{
73-
"finalizer already exists -> no new finalizer",
90+
"persistentvolumeclaims: finalizer already exists -> no new finalizer",
7491
api.SchemeGroupVersion.WithResource("persistentvolumeclaims"),
7592
claimWithFinalizer,
7693
claimWithFinalizer,
7794
claimWithFinalizer.Namespace,
7895
},
7996
{
80-
"create -> add finalizer",
97+
"persistentvolumes: create -> add finalizer",
8198
api.SchemeGroupVersion.WithResource("persistentvolumes"),
8299
pv,
83100
pvWithFinalizer,
84101
pv.Namespace,
85102
},
86103
{
87-
"finalizer already exists -> no new finalizer",
104+
"persistentvolumes: finalizer already exists -> no new finalizer",
88105
api.SchemeGroupVersion.WithResource("persistentvolumes"),
89106
pvWithFinalizer,
90107
pvWithFinalizer,
91108
pvWithFinalizer.Namespace,
92109
},
110+
{
111+
"volumeattributesclasses: create -> add finalizer",
112+
storageapi.SchemeGroupVersion.WithResource("volumeattributesclasses"),
113+
vac,
114+
vacWithFinalizer,
115+
vac.Namespace,
116+
},
117+
{
118+
"volumeattributesclasses: finalizer already exists -> no new finalizer",
119+
storageapi.SchemeGroupVersion.WithResource("volumeattributesclasses"),
120+
vacWithFinalizer,
121+
vacWithFinalizer,
122+
vac.Namespace,
123+
},
93124
}
94125

95126
for _, test := range tests {
96127
t.Run(test.name, func(t *testing.T) {
97128
ctrl := newPlugin()
98129

130+
// Enable feature gate for tests with VolumeAttributesClass
131+
if test.object.GetObjectKind().GroupVersionKind().Kind == "VolumeAttributesClass" {
132+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true)
133+
}
134+
99135
obj := test.object.DeepCopyObject()
100136
attrs := admission.NewAttributesRecord(
101137
obj, // new object

‎test/e2e/storage/testsuites/volume_modify.go‎

Lines changed: 156 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,21 @@ package testsuites
1919
import (
2020
"context"
2121
"encoding/json"
22+
"fmt"
2223
"time"
2324

2425
"github.com/onsi/ginkgo/v2"
2526
"github.com/onsi/gomega"
2627
v1 "k8s.io/api/core/v1"
2728
storagev1beta1 "k8s.io/api/storage/v1beta1"
29+
apierrors "k8s.io/apimachinery/pkg/api/errors"
2830
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2931
"k8s.io/apimachinery/pkg/types"
3032
"k8s.io/apimachinery/pkg/util/errors"
3133
clientset "k8s.io/client-go/kubernetes"
34+
"k8s.io/client-go/util/retry"
3235
"k8s.io/kubernetes/pkg/features"
36+
volumeutil "k8s.io/kubernetes/pkg/volume/util"
3337
"k8s.io/kubernetes/test/e2e/framework"
3438
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
3539
e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
@@ -44,6 +48,7 @@ const (
4448
setVACWaitPeriod = 30 * time.Second
4549
modifyVolumeWaitPeriod = 10 * time.Minute
4650
vacCleanupWaitPeriod = 30 * time.Second
51+
claimDeletingTimeout = 3 * time.Minute
4752
)
4853

4954
type volumeModifyTestSuite struct {
@@ -266,6 +271,126 @@ func (v *volumeModifyTestSuite) DefineTests(driver storageframework.TestDriver,
266271
err = e2epv.WaitForPersistentVolumeClaimModified(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod)
267272
framework.ExpectNoError(err, "While waiting for PVC to have expected VAC")
268273
})
274+
275+
ginkgo.It("should be protected by vac-protection finalizer", func(ctx context.Context) {
276+
init(ctx, false /* volume created without VAC */)
277+
ginkgo.DeferCleanup(cleanup)
278+
279+
vacDriver, _ := driver.(storageframework.VolumeAttributesClassTestDriver)
280+
newVAC := vacDriver.GetVolumeAttributesClass(ctx, l.config)
281+
gomega.Expect(newVAC).NotTo(gomega.BeNil())
282+
createdVAC, err := f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Create(ctx, newVAC, metav1.CreateOptions{})
283+
framework.ExpectNoError(err, "While creating new VolumeAttributesClass")
284+
ginkgo.DeferCleanup(CleanupVAC, newVAC, f.ClientSet, vacCleanupWaitPeriod)
285+
286+
ginkgo.By("Verifying that the vac-protection finalizer is set when it is created")
287+
gomega.Expect(createdVAC.Finalizers).Should(gomega.ContainElement(volumeutil.VACProtectionFinalizer),
288+
"vac-protection finalizer was not set for VolumeAttributesClass %q", createdVAC.Name)
289+
290+
ginkgo.By("Creating a pod with dynamically provisioned volume")
291+
podConfig := e2epod.Config{
292+
NS: f.Namespace.Name,
293+
PVCs: []*v1.PersistentVolumeClaim{l.resource.Pvc},
294+
SeLinuxLabel: e2epod.GetLinuxLabel(),
295+
NodeSelection: l.config.ClientNodeSelection,
296+
ImageID: e2epod.GetDefaultTestImageID(),
297+
}
298+
pod, err := e2epod.CreateSecPodWithNodeSelection(ctx, f.ClientSet, &podConfig, f.Timeouts.PodStart)
299+
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, f.ClientSet, pod)
300+
framework.ExpectNoError(err, "While creating pod for modifying")
301+
302+
ginkgo.By("Modifying PVC via VAC")
303+
l.resource.Pvc = SetPVCVACName(ctx, l.resource.Pvc, newVAC.Name, f.ClientSet, setVACWaitPeriod)
304+
gomega.Expect(l.resource.Pvc).NotTo(gomega.BeNil())
305+
306+
ginkgo.By("Checking PVC status")
307+
err = e2epv.WaitForPersistentVolumeClaimModified(ctx, f.ClientSet, l.resource.Pvc, modifyVolumeWaitPeriod)
308+
framework.ExpectNoError(err, "While waiting for PVC to have expected VAC")
309+
310+
ginkgo.By("Attempting to delete the VolumeAttributesClass should be stuck and the vac-protection finalizer is consistently exists")
311+
err = f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Delete(ctx, newVAC.Name, metav1.DeleteOptions{})
312+
framework.ExpectNoError(err, "Failed to delete VolumeAttributesClass %q", newVAC.Name)
313+
// Check that the finalizer is consistently exists
314+
gomega.Consistently(ctx, framework.GetObject(f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Get, createdVAC.Name, metav1.GetOptions{})).
315+
WithPolling(framework.Poll).WithTimeout(vacCleanupWaitPeriod).Should(gomega.HaveField("Finalizers",
316+
gomega.ContainElement(volumeutil.VACProtectionFinalizer)), "finalizer %s was unexpectedly removed", volumeutil.VACProtectionFinalizer)
317+
318+
ginkgo.By("Deleting the pod gracefully")
319+
err = e2epod.DeletePodWithWait(ctx, f.ClientSet, pod)
320+
framework.ExpectNoError(err, "failed to delete pod")
321+
322+
ginkgo.By("Update the PV reclaim policy to retain")
323+
pvc, err := f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Get(ctx, l.resource.Pvc.Name, metav1.GetOptions{})
324+
framework.ExpectNoError(err, "Failed to get PVC %q", l.resource.Pvc.Name)
325+
pvName := pvc.Spec.VolumeName
326+
pv, err := f.ClientSet.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
327+
framework.ExpectNoError(err, "Failed to get PV %q", pvName)
328+
originPv := pv.DeepCopy()
329+
pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimRetain
330+
_, err = f.ClientSet.CoreV1().PersistentVolumes().Update(ctx, pv, metav1.UpdateOptions{})
331+
ginkgo.DeferCleanup(recoverPvReclaimPolicyAndRemoveClaimRef, f.ClientSet, originPv)
332+
framework.ExpectNoError(err, "Failed to update PV %q reclaim policy", pvName)
333+
334+
// The vac_protection_controller make sure there is a VolumeAttributesClass that is not used by any PVC/PV
335+
// and the VolumeAttributesClass has the vac-protection finalizer, so the VolumeAttributesClass should not be deleted
336+
// after the PVC is deleted since the PVC bound PV which with the same vac is retained.
337+
ginkgo.By(fmt.Sprintf("Deleting PVC %q to make the vac unused for the PVC", newVAC.Name))
338+
err = f.ClientSet.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Delete(ctx, l.resource.Pvc.Name, metav1.DeleteOptions{})
339+
framework.ExpectNoError(err, "Failed to delete PVC %q", l.resource.Pvc.Name)
340+
341+
ginkgo.By(fmt.Sprintf("Waiting for PVC %q to be fully deleted", l.resource.Pvc.Name))
342+
gomega.Eventually(ctx, func() bool {
343+
_, err := f.ClientSet.CoreV1().PersistentVolumeClaims(l.resource.Pvc.Name).Get(ctx, l.resource.Pvc.Name, metav1.GetOptions{})
344+
return apierrors.IsNotFound(err)
345+
}).
346+
WithPolling(framework.Poll).
347+
WithTimeout(claimDeletingTimeout).
348+
Should(gomega.BeTrueBecause("pvc unexpectedly still exists"))
349+
350+
ginkgo.By(fmt.Sprintf("Waiting for PV %q to be released", pvName))
351+
gomega.Eventually(func() v1.PersistentVolumePhase {
352+
pv, err := f.ClientSet.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
353+
if err != nil {
354+
framework.Logf("Failed to get PV %q: %v", pvName, err)
355+
return ""
356+
}
357+
return pv.Status.Phase
358+
}).
359+
WithPolling(framework.Poll).
360+
WithTimeout(claimDeletingTimeout).
361+
Should(gomega.Equal(v1.VolumeReleased))
362+
363+
ginkgo.By(fmt.Sprintf("Checking the vac-protection finalizer is still consistently exists on VolumeAttributesClass %q", newVAC.Name))
364+
gomega.Consistently(ctx, framework.GetObject(f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Get, createdVAC.Name, metav1.GetOptions{})).
365+
WithPolling(framework.Poll).
366+
WithTimeout(vacCleanupWaitPeriod).
367+
Should(gomega.HaveField("Finalizers",
368+
gomega.ContainElement(volumeutil.VACProtectionFinalizer)), "finalizer %s was unexpectedly removed", volumeutil.VACProtectionFinalizer)
369+
370+
ginkgo.By(fmt.Sprintf("Deleting PV %q to make the vac unused for the PV", newVAC.Name))
371+
pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimDelete
372+
recoverPvReclaimPolicyAndRemoveClaimRef(ctx, f.ClientSet, pv)
373+
374+
ginkgo.By(fmt.Sprintf("Waiting for PV %q to be deleted", pvName))
375+
gomega.Eventually(func() bool {
376+
_, err := f.ClientSet.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{})
377+
return apierrors.IsNotFound(err)
378+
}).
379+
WithPolling(framework.Poll).
380+
WithTimeout(claimDeletingTimeout).
381+
Should(gomega.BeTrueBecause("pv unexpectedly still exists"))
382+
383+
ginkgo.By(fmt.Sprintf("Confirming final deletion of VolumeAttributesClass %q", newVAC.Name))
384+
gomega.Eventually(ctx, func(ctx context.Context) bool {
385+
_, err := f.ClientSet.StorageV1beta1().VolumeAttributesClasses().Get(ctx, newVAC.Name, metav1.GetOptions{})
386+
return apierrors.IsNotFound(err)
387+
}).
388+
WithPolling(framework.Poll).
389+
WithTimeout(vacCleanupWaitPeriod).
390+
Should(gomega.BeTrueBecause("vac unexpectedly still exists"))
391+
392+
})
393+
269394
}
270395

271396
// SetPVCVACName sets the VolumeAttributesClassName on a PVC object
@@ -285,6 +410,7 @@ func SetPVCVACName(ctx context.Context, origPVC *v1.PersistentVolumeClaim, name
285410
return patchedPVC
286411
}
287412

413+
// MakeInvalidVAC creates a VolumeAttributesClass with an invalid parameter
288414
func MakeInvalidVAC(config *storageframework.PerTestConfig) *storagev1beta1.VolumeAttributesClass {
289415
return storageframework.CopyVolumeAttributesClass(&storagev1beta1.VolumeAttributesClass{
290416
DriverName: config.GetUniqueDriverName(),
@@ -294,8 +420,37 @@ func MakeInvalidVAC(config *storageframework.PerTestConfig) *storagev1beta1.Volu
294420
}, config.Framework.Namespace.Name, "e2e-vac-invalid")
295421
}
296422

423+
// CleanupVAC cleans up the test VolumeAttributesClass
297424
func CleanupVAC(ctx context.Context, vac *storagev1beta1.VolumeAttributesClass, c clientset.Interface, timeout time.Duration) {
298425
gomega.Eventually(ctx, func() error {
299-
return c.StorageV1beta1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})
426+
err := c.StorageV1beta1().VolumeAttributesClasses().Delete(ctx, vac.Name, metav1.DeleteOptions{})
427+
if apierrors.IsNotFound(err) {
428+
framework.Logf("VolumeAttributesClass %q is already cleaned up", vac.Name)
429+
return nil
430+
}
431+
return err
300432
}, timeout, modifyPollInterval).Should(gomega.BeNil())
301433
}
434+
435+
// recoverPvReclaimPolicyAndRemoveClaimRef recovers the test pv's reclaim policy to expected used for clean up test PV
436+
func recoverPvReclaimPolicyAndRemoveClaimRef(ctx context.Context, c clientset.Interface, expectedPv *v1.PersistentVolume) {
437+
setPvReclaimPolicyErr := retry.RetryOnConflict(retry.DefaultRetry, func() error {
438+
pv, err := c.CoreV1().PersistentVolumes().Get(ctx, expectedPv.Name, metav1.GetOptions{})
439+
if err != nil {
440+
if apierrors.IsNotFound(err) {
441+
framework.Logf("PV %q is already cleaned up", expectedPv.Name)
442+
return nil
443+
}
444+
return err
445+
}
446+
if pv.Spec.PersistentVolumeReclaimPolicy == expectedPv.Spec.PersistentVolumeReclaimPolicy && pv.Spec.ClaimRef == nil {
447+
framework.Logf("PV %q reclaim policy is already recovered to %q", expectedPv.Name, expectedPv.Spec.PersistentVolumeReclaimPolicy)
448+
return nil
449+
}
450+
pv.Spec.ClaimRef = nil
451+
pv.Spec.PersistentVolumeReclaimPolicy = expectedPv.Spec.PersistentVolumeReclaimPolicy
452+
_, err = c.CoreV1().PersistentVolumes().Update(ctx, pv, metav1.UpdateOptions{})
453+
return err
454+
})
455+
framework.ExpectNoError(setPvReclaimPolicyErr, "Failed to set PV %q reclaim policy to %q", expectedPv.Name, expectedPv.Spec.PersistentVolumeReclaimPolicy)
456+
}

0 commit comments

Comments
 (0)