Skip to content

Commit 842fed6

Browse files
committed
Refactor PV scheduling library into separate package
1 parent 0b6c028 commit 842fed6

20 files changed

+588
-407
lines changed

hack/.golint_failures

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,8 @@ pkg/controller/volume/expand
108108
pkg/controller/volume/persistentvolume
109109
pkg/controller/volume/persistentvolume/config/v1alpha1
110110
pkg/controller/volume/persistentvolume/options
111+
pkg/controller/volume/persistentvolume/testing
112+
pkg/controller/volume/scheduling
111113
pkg/credentialprovider
112114
pkg/credentialprovider/gcp
113115
pkg/features

pkg/controller/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ filegroup(
146146
"//pkg/controller/volume/protectionutil:all-srcs",
147147
"//pkg/controller/volume/pvcprotection:all-srcs",
148148
"//pkg/controller/volume/pvprotection:all-srcs",
149+
"//pkg/controller/volume/scheduling:all-srcs",
149150
],
150151
tags = ["automanaged"],
151152
)

pkg/controller/volume/persistentvolume/BUILD

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,6 @@ go_library(
1212
"index.go",
1313
"pv_controller.go",
1414
"pv_controller_base.go",
15-
"scheduler_assume_cache.go",
16-
"scheduler_bind_cache_metrics.go",
17-
"scheduler_binder.go",
18-
"scheduler_binder_cache.go",
19-
"scheduler_binder_fake.go",
20-
"util.go",
2115
"volume_host.go",
2216
],
2317
importpath = "k8s.io/kubernetes/pkg/controller/volume/persistentvolume",
@@ -26,6 +20,7 @@ go_library(
2620
"//pkg/controller:go_default_library",
2721
"//pkg/controller/volume/events:go_default_library",
2822
"//pkg/controller/volume/persistentvolume/metrics:go_default_library",
23+
"//pkg/controller/volume/persistentvolume/util:go_default_library",
2924
"//pkg/features:go_default_library",
3025
"//pkg/util/goroutinemap:go_default_library",
3126
"//pkg/util/goroutinemap/exponentialbackoff:go_default_library",
@@ -39,14 +34,12 @@ go_library(
3934
"//staging/src/k8s.io/api/storage/v1:go_default_library",
4035
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
4136
"//staging/src/k8s.io/apimachinery/pkg/api/meta:go_default_library",
42-
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
4337
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
4438
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
4539
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
4640
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
4741
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
4842
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
49-
"//staging/src/k8s.io/apiserver/pkg/storage/etcd:go_default_library",
5043
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
5144
"//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
5245
"//staging/src/k8s.io/client-go/informers/storage/v1:go_default_library",
@@ -62,7 +55,6 @@ go_library(
6255
"//staging/src/k8s.io/cloud-provider:go_default_library",
6356
"//staging/src/k8s.io/cloud-provider/volume/errors:go_default_library",
6457
"//staging/src/k8s.io/csi-translation-lib:go_default_library",
65-
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
6658
"//vendor/k8s.io/klog:go_default_library",
6759
],
6860
)
@@ -77,15 +69,14 @@ go_test(
7769
"provision_test.go",
7870
"pv_controller_test.go",
7971
"recycle_test.go",
80-
"scheduler_assume_cache_test.go",
81-
"scheduler_binder_cache_test.go",
82-
"scheduler_binder_test.go",
8372
],
8473
embed = [":go_default_library"],
8574
deps = [
8675
"//pkg/api/testapi:go_default_library",
8776
"//pkg/apis/core:go_default_library",
8877
"//pkg/controller:go_default_library",
78+
"//pkg/controller/volume/persistentvolume/testing:go_default_library",
79+
"//pkg/controller/volume/persistentvolume/util:go_default_library",
8980
"//pkg/features:go_default_library",
9081
"//pkg/volume:go_default_library",
9182
"//pkg/volume/util:go_default_library",
@@ -95,16 +86,12 @@ go_test(
9586
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
9687
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
9788
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
98-
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
99-
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
10089
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
101-
"//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
10290
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
10391
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
10492
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
10593
"//staging/src/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
10694
"//staging/src/k8s.io/client-go/informers:go_default_library",
107-
"//staging/src/k8s.io/client-go/informers/core/v1:go_default_library",
10895
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
10996
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
11097
"//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library",
@@ -132,6 +119,8 @@ filegroup(
132119
"//pkg/controller/volume/persistentvolume/config:all-srcs",
133120
"//pkg/controller/volume/persistentvolume/metrics:all-srcs",
134121
"//pkg/controller/volume/persistentvolume/options:all-srcs",
122+
"//pkg/controller/volume/persistentvolume/testing:all-srcs",
123+
"//pkg/controller/volume/persistentvolume/util:all-srcs",
135124
],
136125
tags = ["automanaged"],
137126
)

pkg/controller/volume/persistentvolume/index.go

Lines changed: 2 additions & 192 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,9 @@ import (
2121
"sort"
2222

2323
"k8s.io/api/core/v1"
24-
"k8s.io/apimachinery/pkg/api/resource"
25-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
26-
"k8s.io/apimachinery/pkg/labels"
27-
utilfeature "k8s.io/apiserver/pkg/util/feature"
2824
"k8s.io/client-go/tools/cache"
2925
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
30-
"k8s.io/kubernetes/pkg/features"
26+
pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util"
3127
volumeutil "k8s.io/kubernetes/pkg/volume/util"
3228
)
3329

@@ -96,7 +92,7 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *v1.PersistentVol
9692
return nil, err
9793
}
9894

99-
bestVol, err := findMatchingVolume(claim, volumes, nil /* node for topology binding*/, nil /* exclusion map */, delayBinding)
95+
bestVol, err := pvutil.FindMatchingVolume(claim, volumes, nil /* node for topology binding*/, nil /* exclusion map */, delayBinding)
10096
if err != nil {
10197
return nil, err
10298
}
@@ -108,176 +104,6 @@ func (pvIndex *persistentVolumeOrderedIndex) findByClaim(claim *v1.PersistentVol
108104
return nil, nil
109105
}
110106

111-
// findMatchingVolume goes through the list of volumes to find the best matching volume
112-
// for the claim.
113-
//
114-
// This function is used by both the PV controller and scheduler.
115-
//
116-
// delayBinding is true only in the PV controller path. When set, prebound PVs are still returned
117-
// as a match for the claim, but unbound PVs are skipped.
118-
//
119-
// node is set only in the scheduler path. When set, the PV node affinity is checked against
120-
// the node's labels.
121-
//
122-
// excludedVolumes is only used in the scheduler path, and is needed for evaluating multiple
123-
// unbound PVCs for a single Pod at one time. As each PVC finds a matching PV, the chosen
124-
// PV needs to be excluded from future matching.
125-
func findMatchingVolume(
126-
claim *v1.PersistentVolumeClaim,
127-
volumes []*v1.PersistentVolume,
128-
node *v1.Node,
129-
excludedVolumes map[string]*v1.PersistentVolume,
130-
delayBinding bool) (*v1.PersistentVolume, error) {
131-
132-
var smallestVolume *v1.PersistentVolume
133-
var smallestVolumeQty resource.Quantity
134-
requestedQty := claim.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
135-
requestedClass := v1helper.GetPersistentVolumeClaimClass(claim)
136-
137-
var selector labels.Selector
138-
if claim.Spec.Selector != nil {
139-
internalSelector, err := metav1.LabelSelectorAsSelector(claim.Spec.Selector)
140-
if err != nil {
141-
// should be unreachable code due to validation
142-
return nil, fmt.Errorf("error creating internal label selector for claim: %v: %v", claimToClaimKey(claim), err)
143-
}
144-
selector = internalSelector
145-
}
146-
147-
// Go through all available volumes with two goals:
148-
// - find a volume that is either pre-bound by user or dynamically
149-
// provisioned for this claim. Because of this we need to loop through
150-
// all volumes.
151-
// - find the smallest matching one if there is no volume pre-bound to
152-
// the claim.
153-
for _, volume := range volumes {
154-
if _, ok := excludedVolumes[volume.Name]; ok {
155-
// Skip volumes in the excluded list
156-
continue
157-
}
158-
159-
volumeQty := volume.Spec.Capacity[v1.ResourceStorage]
160-
161-
// check if volumeModes do not match (feature gate protected)
162-
isMismatch, err := checkVolumeModeMismatches(&claim.Spec, &volume.Spec)
163-
if err != nil {
164-
return nil, fmt.Errorf("error checking if volumeMode was a mismatch: %v", err)
165-
}
166-
// filter out mismatching volumeModes
167-
if isMismatch {
168-
continue
169-
}
170-
171-
// check if PV's DeletionTimeStamp is set, if so, skip this volume.
172-
if utilfeature.DefaultFeatureGate.Enabled(features.StorageObjectInUseProtection) {
173-
if volume.ObjectMeta.DeletionTimestamp != nil {
174-
continue
175-
}
176-
}
177-
178-
nodeAffinityValid := true
179-
if node != nil {
180-
// Scheduler path, check that the PV NodeAffinity
181-
// is satisfied by the node
182-
err := volumeutil.CheckNodeAffinity(volume, node.Labels)
183-
if err != nil {
184-
nodeAffinityValid = false
185-
}
186-
}
187-
188-
if IsVolumeBoundToClaim(volume, claim) {
189-
// this claim and volume are pre-bound; return
190-
// the volume if the size request is satisfied,
191-
// otherwise continue searching for a match
192-
if volumeQty.Cmp(requestedQty) < 0 {
193-
continue
194-
}
195-
196-
// If PV node affinity is invalid, return no match.
197-
// This means the prebound PV (and therefore PVC)
198-
// is not suitable for this node.
199-
if !nodeAffinityValid {
200-
return nil, nil
201-
}
202-
203-
return volume, nil
204-
}
205-
206-
if node == nil && delayBinding {
207-
// PV controller does not bind this claim.
208-
// Scheduler will handle binding unbound volumes
209-
// Scheduler path will have node != nil
210-
continue
211-
}
212-
213-
// filter out:
214-
// - volumes in non-available phase
215-
// - volumes bound to another claim
216-
// - volumes whose labels don't match the claim's selector, if specified
217-
// - volumes in Class that is not requested
218-
// - volumes whose NodeAffinity does not match the node
219-
if volume.Status.Phase != v1.VolumeAvailable {
220-
// We ignore volumes in non-available phase, because volumes that
221-
// satisfies matching criteria will be updated to available, binding
222-
// them now has high chance of encountering unnecessary failures
223-
// due to API conflicts.
224-
continue
225-
} else if volume.Spec.ClaimRef != nil {
226-
continue
227-
} else if selector != nil && !selector.Matches(labels.Set(volume.Labels)) {
228-
continue
229-
}
230-
if v1helper.GetPersistentVolumeClass(volume) != requestedClass {
231-
continue
232-
}
233-
if !nodeAffinityValid {
234-
continue
235-
}
236-
237-
if node != nil {
238-
// Scheduler path
239-
// Check that the access modes match
240-
if !checkAccessModes(claim, volume) {
241-
continue
242-
}
243-
}
244-
245-
if volumeQty.Cmp(requestedQty) >= 0 {
246-
if smallestVolume == nil || smallestVolumeQty.Cmp(volumeQty) > 0 {
247-
smallestVolume = volume
248-
smallestVolumeQty = volumeQty
249-
}
250-
}
251-
}
252-
253-
if smallestVolume != nil {
254-
// Found a matching volume
255-
return smallestVolume, nil
256-
}
257-
258-
return nil, nil
259-
}
260-
261-
// checkVolumeModeMismatches is a convenience method that checks volumeMode for PersistentVolume
262-
// and PersistentVolumeClaims
263-
func checkVolumeModeMismatches(pvcSpec *v1.PersistentVolumeClaimSpec, pvSpec *v1.PersistentVolumeSpec) (bool, error) {
264-
if !utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
265-
return false, nil
266-
}
267-
268-
// In HA upgrades, we cannot guarantee that the apiserver is on a version >= controller-manager.
269-
// So we default a nil volumeMode to filesystem
270-
requestedVolumeMode := v1.PersistentVolumeFilesystem
271-
if pvcSpec.VolumeMode != nil {
272-
requestedVolumeMode = *pvcSpec.VolumeMode
273-
}
274-
pvVolumeMode := v1.PersistentVolumeFilesystem
275-
if pvSpec.VolumeMode != nil {
276-
pvVolumeMode = *pvSpec.VolumeMode
277-
}
278-
return requestedVolumeMode != pvVolumeMode, nil
279-
}
280-
281107
// findBestMatchForClaim is a convenience method that finds a volume by the claim's AccessModes and requests for Storage
282108
func (pvIndex *persistentVolumeOrderedIndex) findBestMatchForClaim(claim *v1.PersistentVolumeClaim, delayBinding bool) (*v1.PersistentVolume, error) {
283109
return pvIndex.findByClaim(claim, delayBinding)
@@ -362,19 +188,3 @@ func claimToClaimKey(claim *v1.PersistentVolumeClaim) string {
362188
func claimrefToClaimKey(claimref *v1.ObjectReference) string {
363189
return fmt.Sprintf("%s/%s", claimref.Namespace, claimref.Name)
364190
}
365-
366-
// Returns true if PV satisfies all the PVC's requested AccessModes
367-
func checkAccessModes(claim *v1.PersistentVolumeClaim, volume *v1.PersistentVolume) bool {
368-
pvModesMap := map[v1.PersistentVolumeAccessMode]bool{}
369-
for _, mode := range volume.Spec.AccessModes {
370-
pvModesMap[mode] = true
371-
}
372-
373-
for _, mode := range claim.Spec.AccessModes {
374-
_, ok := pvModesMap[mode]
375-
if !ok {
376-
return false
377-
}
378-
}
379-
return true
380-
}

0 commit comments

Comments
 (0)