Skip to content

Commit 68899f8

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#124360 from carlory/kep-3751-quota-2
Add quota support for PVC with VolumeAttributesClass
2 parents 336a32a + 27706a0 commit 68899f8

File tree

9 files changed

+1004
-19
lines changed

9 files changed

+1004
-19
lines changed

‎pkg/apis/core/helper/helpers.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ var standardResourceQuotaScopes = sets.New(
119119
core.ResourceQuotaScopeBestEffort,
120120
core.ResourceQuotaScopeNotBestEffort,
121121
core.ResourceQuotaScopePriorityClass,
122+
core.ResourceQuotaScopeVolumeAttributesClass,
122123
)
123124

124125
// IsStandardResourceQuotaScope returns true if the scope is a standard value
@@ -139,6 +140,14 @@ var podComputeQuotaResources = sets.New(
139140
core.ResourceRequestsMemory,
140141
)
141142

143+
var pvcObjectCountQuotaResources = sets.New(
144+
core.ResourcePersistentVolumeClaims,
145+
)
146+
147+
var pvcStorageQuotaResources = sets.New(
148+
core.ResourceRequestsStorage,
149+
)
150+
142151
// IsResourceQuotaScopeValidForResource returns true if the resource applies to the specified scope
143152
func IsResourceQuotaScopeValidForResource(scope core.ResourceQuotaScope, resource core.ResourceName) bool {
144153
switch scope {
@@ -147,6 +156,8 @@ func IsResourceQuotaScopeValidForResource(scope core.ResourceQuotaScope, resourc
147156
return podObjectCountQuotaResources.Has(resource) || podComputeQuotaResources.Has(resource)
148157
case core.ResourceQuotaScopeBestEffort:
149158
return podObjectCountQuotaResources.Has(resource)
159+
case core.ResourceQuotaScopeVolumeAttributesClass:
160+
return pvcObjectCountQuotaResources.Has(resource) || pvcStorageQuotaResources.Has(resource)
150161
default:
151162
return true
152163
}

‎pkg/apis/core/types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6048,6 +6048,9 @@ const (
60486048
ResourceQuotaScopePriorityClass ResourceQuotaScope = "PriorityClass"
60496049
// Match all pod objects that have cross-namespace pod (anti)affinity mentioned
60506050
ResourceQuotaScopeCrossNamespacePodAffinity ResourceQuotaScope = "CrossNamespacePodAffinity"
6051+
6052+
// Match all pvc objects that have volume attributes class mentioned.
6053+
ResourceQuotaScopeVolumeAttributesClass ResourceQuotaScope = "VolumeAttributesClass"
60516054
)
60526055

60536056
// ResourceQuotaSpec defines the desired hard limits to enforce for Quota

‎pkg/generated/openapi/zz_generated.openapi.go‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/quota/v1/evaluator/core/persistent_volume_claims.go‎

Lines changed: 113 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,20 @@ import (
2222

2323
corev1 "k8s.io/api/core/v1"
2424
"k8s.io/apimachinery/pkg/api/resource"
25+
"k8s.io/apimachinery/pkg/labels"
2526
"k8s.io/apimachinery/pkg/runtime"
2627
"k8s.io/apimachinery/pkg/runtime/schema"
28+
"k8s.io/apimachinery/pkg/util/sets"
2729
"k8s.io/apiserver/pkg/admission"
2830
quota "k8s.io/apiserver/pkg/quota/v1"
2931
"k8s.io/apiserver/pkg/quota/v1/generic"
3032
utilfeature "k8s.io/apiserver/pkg/util/feature"
3133
storagehelpers "k8s.io/component-helpers/storage/volume"
3234
api "k8s.io/kubernetes/pkg/apis/core"
3335
k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1"
36+
"k8s.io/kubernetes/pkg/apis/core/v1/helper"
3437
k8sfeatures "k8s.io/kubernetes/pkg/features"
38+
"k8s.io/utils/ptr"
3539
)
3640

3741
// the name used for object count quota
@@ -90,26 +94,68 @@ func (p *pvcEvaluator) GroupResource() schema.GroupResource {
9094

9195
// Handles returns true if the evaluator should handle the specified operation.
9296
func (p *pvcEvaluator) Handles(a admission.Attributes) bool {
93-
if a.GetSubresource() != "" {
97+
op := a.GetOperation()
98+
switch a.GetSubresource() {
99+
case "":
100+
return op == admission.Create || op == admission.Update
101+
case "status":
102+
pvc, err1 := toExternalPersistentVolumeClaimOrError(a.GetObject())
103+
oldPVC, err2 := toExternalPersistentVolumeClaimOrError(a.GetOldObject())
104+
if err1 != nil || err2 != nil {
105+
return false
106+
}
107+
return RequiresQuotaReplenish(pvc, oldPVC)
108+
default:
94109
return false
95110
}
96-
op := a.GetOperation()
97-
return admission.Create == op || admission.Update == op
98111
}
99112

100113
// Matches returns true if the evaluator matches the specified quota with the provided input item
101114
func (p *pvcEvaluator) Matches(resourceQuota *corev1.ResourceQuota, item runtime.Object) (bool, error) {
115+
if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) {
116+
return generic.Matches(resourceQuota, item, p.MatchingResources, pvcMatchesScopeFunc)
117+
}
102118
return generic.Matches(resourceQuota, item, p.MatchingResources, generic.MatchesNoScopeFunc)
103119
}
104120

105121
// MatchingScopes takes the input specified list of scopes and input object. Returns the set of scopes resource matches.
106-
func (p *pvcEvaluator) MatchingScopes(item runtime.Object, scopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) {
122+
func (p *pvcEvaluator) MatchingScopes(item runtime.Object, scopeSelectors []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) {
123+
if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) {
124+
matchedScopes := []corev1.ScopedResourceSelectorRequirement{}
125+
for _, selector := range scopeSelectors {
126+
match, err := pvcMatchesScopeFunc(selector, item)
127+
if err != nil {
128+
return []corev1.ScopedResourceSelectorRequirement{}, fmt.Errorf("error on matching scope %v: %w", selector, err)
129+
}
130+
if match {
131+
matchedScopes = append(matchedScopes, selector)
132+
}
133+
}
134+
return matchedScopes, nil
135+
}
107136
return []corev1.ScopedResourceSelectorRequirement{}, nil
108137
}
109138

110139
// UncoveredQuotaScopes takes the input matched scopes which are limited by configuration and the matched quota scopes.
111140
// It returns the scopes which are in limited scopes but don't have a corresponding covering quota scope
112141
func (p *pvcEvaluator) UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourceSelectorRequirement, matchedQuotaScopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error) {
142+
if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) {
143+
uncoveredScopes := []corev1.ScopedResourceSelectorRequirement{}
144+
for _, selector := range limitedScopes {
145+
isCovered := false
146+
for _, matchedScopeSelector := range matchedQuotaScopes {
147+
if matchedScopeSelector.ScopeName == selector.ScopeName {
148+
isCovered = true
149+
break
150+
}
151+
}
152+
153+
if !isCovered {
154+
uncoveredScopes = append(uncoveredScopes, selector)
155+
}
156+
}
157+
return uncoveredScopes, nil
158+
}
113159
return []corev1.ScopedResourceSelectorRequirement{}, nil
114160
}
115161

@@ -202,6 +248,9 @@ func (p *pvcEvaluator) getStorageUsage(pvc *corev1.PersistentVolumeClaim) *resou
202248

203249
// UsageStats calculates aggregate usage for the object.
204250
func (p *pvcEvaluator) UsageStats(options quota.UsageStatsOptions) (quota.UsageStats, error) {
251+
if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) {
252+
return generic.CalculateUsageStats(options, p.listFuncByNamespace, pvcMatchesScopeFunc, p.Usage)
253+
}
205254
return generic.CalculateUsageStats(options, p.listFuncByNamespace, generic.MatchesNoScopeFunc, p.Usage)
206255
}
207256

@@ -230,5 +279,65 @@ func RequiresQuotaReplenish(pvc, oldPVC *corev1.PersistentVolumeClaim) bool {
230279
return true
231280
}
232281
}
282+
if utilfeature.DefaultFeatureGate.Enabled(k8sfeatures.VolumeAttributesClass) {
283+
oldNames := getReferencedVolumeAttributesClassNames(oldPVC)
284+
newNames := getReferencedVolumeAttributesClassNames(pvc)
285+
if !oldNames.Equal(newNames) {
286+
return true
287+
}
288+
}
233289
return false
234290
}
291+
292+
// pvcMatchesScopeFunc is a function that knows how to evaluate if a pvc matches a scope
293+
func pvcMatchesScopeFunc(selector corev1.ScopedResourceSelectorRequirement, object runtime.Object) (bool, error) {
294+
pvc, err := toExternalPersistentVolumeClaimOrError(object)
295+
if err != nil {
296+
return false, err
297+
}
298+
299+
if selector.ScopeName == corev1.ResourceQuotaScopeVolumeAttributesClass {
300+
if selector.Operator == corev1.ScopeSelectorOpExists {
301+
// This is just checking for existence of a volumeAttributesClass on the pvc,
302+
// no need to take the overhead of selector parsing/evaluation.
303+
vacNames := getReferencedVolumeAttributesClassNames(pvc)
304+
return len(vacNames) != 0, nil
305+
}
306+
return pvcMatchesSelector(pvc, selector)
307+
}
308+
return false, nil
309+
}
310+
311+
func pvcMatchesSelector(pvc *corev1.PersistentVolumeClaim, selector corev1.ScopedResourceSelectorRequirement) (bool, error) {
312+
labelSelector, err := helper.ScopedResourceSelectorRequirementsAsSelector(selector)
313+
if err != nil {
314+
return false, fmt.Errorf("failed to parse and convert selector: %w", err)
315+
}
316+
317+
vacNames := getReferencedVolumeAttributesClassNames(pvc)
318+
if len(vacNames) == 0 {
319+
return labelSelector.Matches(labels.Set{}), nil
320+
}
321+
for vacName := range vacNames {
322+
m := labels.Set{string(corev1.ResourceQuotaScopeVolumeAttributesClass): vacName}
323+
if labelSelector.Matches(m) {
324+
return true, nil
325+
}
326+
}
327+
return false, nil
328+
}
329+
330+
func getReferencedVolumeAttributesClassNames(pvc *corev1.PersistentVolumeClaim) sets.Set[string] {
331+
vacNames := sets.New[string]()
332+
if len(ptr.Deref(pvc.Spec.VolumeAttributesClassName, "")) != 0 {
333+
vacNames.Insert(*pvc.Spec.VolumeAttributesClassName)
334+
}
335+
if len(ptr.Deref(pvc.Status.CurrentVolumeAttributesClassName, "")) != 0 {
336+
vacNames.Insert(*pvc.Status.CurrentVolumeAttributesClassName)
337+
}
338+
modifyStatus := pvc.Status.ModifyVolumeStatus
339+
if modifyStatus != nil && len(modifyStatus.TargetVolumeAttributesClassName) != 0 {
340+
vacNames.Insert(modifyStatus.TargetVolumeAttributesClassName)
341+
}
342+
return vacNames
343+
}

‎pkg/quota/v1/evaluator/core/persistent_volume_claims_test.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"reflect"
2121
"testing"
2222

23+
"github.com/google/go-cmp/cmp"
2324
corev1 "k8s.io/api/core/v1"
2425
"k8s.io/apimachinery/pkg/api/resource"
2526
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,6 +32,7 @@ import (
3132
featuregatetesting "k8s.io/component-base/featuregate/testing"
3233
"k8s.io/kubernetes/pkg/apis/core"
3334
"k8s.io/kubernetes/pkg/features"
35+
"k8s.io/utils/ptr"
3436
)
3537

3638
func testVolumeClaim(name string, namespace string, spec core.PersistentVolumeClaimSpec) *core.PersistentVolumeClaim {
@@ -40,6 +42,122 @@ func testVolumeClaim(name string, namespace string, spec core.PersistentVolumeCl
4042
}
4143
}
4244

45+
func TestPersistentVolumeClaimEvaluatorMatchingScopes(t *testing.T) {
46+
evaluator := NewPersistentVolumeClaimEvaluator(nil)
47+
testCases := map[string]struct {
48+
claim *core.PersistentVolumeClaim
49+
selectors []corev1.ScopedResourceSelectorRequirement
50+
wantSelectors []corev1.ScopedResourceSelectorRequirement
51+
}{
52+
"EmptyPVC": {
53+
claim: &core.PersistentVolumeClaim{},
54+
selectors: []corev1.ScopedResourceSelectorRequirement{
55+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist},
56+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists},
57+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}},
58+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
59+
},
60+
wantSelectors: []corev1.ScopedResourceSelectorRequirement{
61+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist},
62+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
63+
},
64+
},
65+
"VolumeAttributesClass": {
66+
claim: &core.PersistentVolumeClaim{
67+
Spec: core.PersistentVolumeClaimSpec{
68+
VolumeAttributesClassName: ptr.To("class1"),
69+
},
70+
},
71+
selectors: []corev1.ScopedResourceSelectorRequirement{
72+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist},
73+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists},
74+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}},
75+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class4"}},
76+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
77+
},
78+
wantSelectors: []corev1.ScopedResourceSelectorRequirement{
79+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists},
80+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}},
81+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
82+
},
83+
},
84+
"VolumeAttributesClassWithTarget": {
85+
claim: &core.PersistentVolumeClaim{
86+
Spec: core.PersistentVolumeClaimSpec{
87+
VolumeAttributesClassName: ptr.To("class1"),
88+
},
89+
Status: core.PersistentVolumeClaimStatus{
90+
CurrentVolumeAttributesClassName: ptr.To("class2"),
91+
},
92+
},
93+
selectors: []corev1.ScopedResourceSelectorRequirement{
94+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist},
95+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists},
96+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}},
97+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}},
98+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2"}},
99+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class4"}},
100+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class4"}},
101+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
102+
},
103+
wantSelectors: []corev1.ScopedResourceSelectorRequirement{
104+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists},
105+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}},
106+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}},
107+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2"}},
108+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class4"}},
109+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
110+
},
111+
},
112+
"VolumeAttributesClassWithModityStatus": {
113+
claim: &core.PersistentVolumeClaim{
114+
Spec: core.PersistentVolumeClaimSpec{
115+
VolumeAttributesClassName: ptr.To("class1"),
116+
},
117+
Status: core.PersistentVolumeClaimStatus{
118+
CurrentVolumeAttributesClassName: ptr.To("class2"),
119+
ModifyVolumeStatus: &core.ModifyVolumeStatus{
120+
TargetVolumeAttributesClassName: "class3",
121+
},
122+
},
123+
},
124+
selectors: []corev1.ScopedResourceSelectorRequirement{
125+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpDoesNotExist},
126+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists},
127+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}},
128+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}},
129+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class3"}},
130+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3"}},
131+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3", "class4"}},
132+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class4"}},
133+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
134+
},
135+
wantSelectors: []corev1.ScopedResourceSelectorRequirement{
136+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpExists},
137+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1"}},
138+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class2"}},
139+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class3"}},
140+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3"}},
141+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpIn, Values: []string{"class1", "class2", "class3", "class4"}},
142+
{ScopeName: corev1.ResourceQuotaScopeVolumeAttributesClass, Operator: corev1.ScopeSelectorOpNotIn, Values: []string{"class4"}},
143+
},
144+
},
145+
}
146+
147+
for testName, testCase := range testCases {
148+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true)
149+
t.Run(testName, func(t *testing.T) {
150+
gotSelectors, err := evaluator.MatchingScopes(testCase.claim, testCase.selectors)
151+
if err != nil {
152+
t.Error(err)
153+
}
154+
if diff := cmp.Diff(testCase.wantSelectors, gotSelectors); diff != "" {
155+
t.Errorf("%v: unexpected diff (-want, +got):\n%s", testName, diff)
156+
}
157+
})
158+
}
159+
}
160+
43161
func TestPersistentVolumeClaimEvaluatorUsage(t *testing.T) {
44162
classGold := "gold"
45163
validClaim := testVolumeClaim("foo", "ns", core.PersistentVolumeClaimSpec{

0 commit comments

Comments
 (0)