Skip to content

Commit 185ba08

Browse files
Huang-WeiHuang-Wei
committed
Move podPassesBasicChecks() to VolumeBinding plugin
1 parent 42cfda2 commit 185ba08

File tree

7 files changed

+73
-69
lines changed

7 files changed

+73
-69
lines changed

‎pkg/scheduler/core/BUILD‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ go_library(
1010
visibility = ["//visibility:public"],
1111
deps = [
1212
"//pkg/api/v1/pod:go_default_library",
13-
"//pkg/features:go_default_library",
1413
"//pkg/scheduler/apis/config:go_default_library",
1514
"//pkg/scheduler/framework/runtime:go_default_library",
1615
"//pkg/scheduler/framework/v1alpha1:go_default_library",
@@ -20,11 +19,8 @@ go_library(
2019
"//pkg/scheduler/profile:go_default_library",
2120
"//pkg/scheduler/util:go_default_library",
2221
"//staging/src/k8s.io/api/core/v1:go_default_library",
23-
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
2422
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
2523
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
26-
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
27-
"//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
2824
"//staging/src/k8s.io/client-go/rest:go_default_library",
2925
"//staging/src/k8s.io/kube-scheduler/extender/v1:go_default_library",
3026
"//vendor/k8s.io/klog/v2:go_default_library",
@@ -40,12 +36,14 @@ go_test(
4036
],
4137
embed = [":go_default_library"],
4238
deps = [
39+
"//pkg/controller/volume/persistentvolume/util:go_default_library",
4340
"//pkg/scheduler/apis/config:go_default_library",
4441
"//pkg/scheduler/framework/plugins/defaultbinder:go_default_library",
4542
"//pkg/scheduler/framework/plugins/noderesources:go_default_library",
4643
"//pkg/scheduler/framework/plugins/podtopologyspread:go_default_library",
4744
"//pkg/scheduler/framework/plugins/queuesort:go_default_library",
4845
"//pkg/scheduler/framework/plugins/selectorspread:go_default_library",
46+
"//pkg/scheduler/framework/plugins/volumebinding:go_default_library",
4947
"//pkg/scheduler/framework/runtime:go_default_library",
5048
"//pkg/scheduler/framework/v1alpha1:go_default_library",
5149
"//pkg/scheduler/internal/cache:go_default_library",

‎pkg/scheduler/core/generic_scheduler.go‎

Lines changed: 0 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,8 @@ import (
2929
"k8s.io/klog/v2"
3030

3131
v1 "k8s.io/api/core/v1"
32-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33-
utilfeature "k8s.io/apiserver/pkg/util/feature"
34-
corelisters "k8s.io/client-go/listers/core/v1"
3532
extenderv1 "k8s.io/kube-scheduler/extender/v1"
3633
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
37-
"k8s.io/kubernetes/pkg/features"
3834
"k8s.io/kubernetes/pkg/scheduler/framework/runtime"
3935
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
4036
internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
@@ -137,12 +133,6 @@ func (g *genericScheduler) Schedule(ctx context.Context, prof *profile.Profile,
137133
trace := utiltrace.New("Scheduling", utiltrace.Field{Key: "namespace", Value: pod.Namespace}, utiltrace.Field{Key: "name", Value: pod.Name})
138134
defer trace.LogIfLong(100 * time.Millisecond)
139135

140-
pvcLister := prof.SharedInformerFactory().Core().V1().PersistentVolumeClaims().Lister()
141-
if err := podPassesBasicChecks(pod, pvcLister); err != nil {
142-
return result, err
143-
}
144-
trace.Step("Basic checks done")
145-
146136
if err := g.snapshot(); err != nil {
147137
return result, err
148138
}
@@ -273,7 +263,6 @@ func (g *genericScheduler) findNodesThatFitPod(ctx context.Context, prof *profil
273263
filteredNodesStatuses[n.Node().Name] = s
274264
}
275265
return nil, filteredNodesStatuses, nil
276-
277266
}
278267

279268
feasibleNodes, err := g.findNodesThatPassFilters(ctx, prof, state, pod, filteredNodesStatuses)
@@ -584,45 +573,6 @@ func (g *genericScheduler) prioritizeNodes(
584573
return result, nil
585574
}
586575

587-
// podPassesBasicChecks makes sanity checks on the pod if it can be scheduled.
588-
func podPassesBasicChecks(pod *v1.Pod, pvcLister corelisters.PersistentVolumeClaimLister) error {
589-
// Check PVCs used by the pod
590-
namespace := pod.Namespace
591-
manifest := &(pod.Spec)
592-
for i := range manifest.Volumes {
593-
volume := &manifest.Volumes[i]
594-
var pvcName string
595-
ephemeral := false
596-
switch {
597-
case volume.PersistentVolumeClaim != nil:
598-
pvcName = volume.PersistentVolumeClaim.ClaimName
599-
case volume.Ephemeral != nil &&
600-
utilfeature.DefaultFeatureGate.Enabled(features.GenericEphemeralVolume):
601-
pvcName = pod.Name + "-" + volume.Name
602-
ephemeral = true
603-
default:
604-
// Volume is not using a PVC, ignore
605-
continue
606-
}
607-
pvc, err := pvcLister.PersistentVolumeClaims(namespace).Get(pvcName)
608-
if err != nil {
609-
// The error has already enough context ("persistentvolumeclaim "myclaim" not found")
610-
return err
611-
}
612-
613-
if pvc.DeletionTimestamp != nil {
614-
return fmt.Errorf("persistentvolumeclaim %q is being deleted", pvc.Name)
615-
}
616-
617-
if ephemeral &&
618-
!metav1.IsControlledBy(pvc, pod) {
619-
return fmt.Errorf("persistentvolumeclaim %q was not created for the pod", pvc.Name)
620-
}
621-
}
622-
623-
return nil
624-
}
625-
626576
// NewGenericScheduler creates a genericScheduler object.
627577
func NewGenericScheduler(
628578
cache internalcache.Cache,

‎pkg/scheduler/core/generic_scheduler_test.go‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"fmt"
2222
"math"
2323
"strconv"
24+
"strings"
2425
"testing"
2526
"time"
2627

@@ -33,12 +34,14 @@ import (
3334
"k8s.io/apimachinery/pkg/util/wait"
3435
"k8s.io/client-go/informers"
3536
clientsetfake "k8s.io/client-go/kubernetes/fake"
37+
pvutil "k8s.io/kubernetes/pkg/controller/volume/persistentvolume/util"
3638
schedulerapi "k8s.io/kubernetes/pkg/scheduler/apis/config"
3739
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder"
3840
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
3941
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/podtopologyspread"
4042
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort"
4143
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/selectorspread"
44+
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding"
4245
frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime"
4346
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
4447
internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
@@ -416,11 +419,17 @@ func TestGenericScheduler(t *testing.T) {
416419
// Pod with existing PVC
417420
registerPlugins: []st.RegisterPluginFunc{
418421
st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
422+
st.RegisterPreFilterPlugin(volumebinding.Name, volumebinding.New),
419423
st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin),
420424
st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
421425
},
422426
nodes: []string{"machine1", "machine2"},
423-
pvcs: []v1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "existingPVC", UID: types.UID("existingPVC"), Namespace: v1.NamespaceDefault}}},
427+
pvcs: []v1.PersistentVolumeClaim{
428+
{
429+
ObjectMeta: metav1.ObjectMeta{Name: "existingPVC", UID: types.UID("existingPVC"), Namespace: v1.NamespaceDefault},
430+
Spec: v1.PersistentVolumeClaimSpec{VolumeName: "existingPV"},
431+
},
432+
},
424433
pod: &v1.Pod{
425434
ObjectMeta: metav1.ObjectMeta{Name: "ignore", UID: types.UID("ignore"), Namespace: v1.NamespaceDefault},
426435
Spec: v1.PodSpec{
@@ -443,6 +452,7 @@ func TestGenericScheduler(t *testing.T) {
443452
// Pod with non existing PVC
444453
registerPlugins: []st.RegisterPluginFunc{
445454
st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
455+
st.RegisterPreFilterPlugin(volumebinding.Name, volumebinding.New),
446456
st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin),
447457
st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
448458
},
@@ -468,6 +478,7 @@ func TestGenericScheduler(t *testing.T) {
468478
// Pod with deleting PVC
469479
registerPlugins: []st.RegisterPluginFunc{
470480
st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
481+
st.RegisterPreFilterPlugin(volumebinding.Name, volumebinding.New),
471482
st.RegisterFilterPlugin("TrueFilter", st.NewTrueFilterPlugin),
472483
st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
473484
},
@@ -726,10 +737,16 @@ func TestGenericScheduler(t *testing.T) {
726737
cache.AddNode(node)
727738
}
728739

740+
ctx := context.Background()
729741
cs := clientsetfake.NewSimpleClientset()
730742
informerFactory := informers.NewSharedInformerFactory(cs, 0)
731-
for i := range test.pvcs {
732-
informerFactory.Core().V1().PersistentVolumeClaims().Informer().GetStore().Add(&test.pvcs[i])
743+
for _, pvc := range test.pvcs {
744+
metav1.SetMetaDataAnnotation(&pvc.ObjectMeta, pvutil.AnnBindCompleted, "true")
745+
cs.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(ctx, &pvc, metav1.CreateOptions{})
746+
if pvName := pvc.Spec.VolumeName; pvName != "" {
747+
pv := v1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: pvName}}
748+
cs.CoreV1().PersistentVolumes().Create(ctx, &pv, metav1.CreateOptions{})
749+
}
733750
}
734751
snapshot := internalcache.NewSnapshot(test.pods, nodes)
735752
fwk, err := st.NewFramework(
@@ -749,9 +766,13 @@ func TestGenericScheduler(t *testing.T) {
749766
cache,
750767
snapshot,
751768
[]framework.Extender{},
752-
schedulerapi.DefaultPercentageOfNodesToScore)
753-
result, err := scheduler.Schedule(context.Background(), prof, framework.NewCycleState(), test.pod)
754-
if err != test.wErr && err.Error() != test.wErr.Error() {
769+
schedulerapi.DefaultPercentageOfNodesToScore,
770+
)
771+
informerFactory.Start(ctx.Done())
772+
informerFactory.WaitForCacheSync(ctx.Done())
773+
774+
result, err := scheduler.Schedule(ctx, prof, framework.NewCycleState(), test.pod)
775+
if err != test.wErr && !strings.Contains(err.Error(), test.wErr.Error()) {
755776
t.Errorf("Unexpected error: %v, expected: %v", err.Error(), test.wErr)
756777
}
757778
if test.expectedHosts != nil && !test.expectedHosts.Has(result.SuggestedHost) {

‎pkg/scheduler/framework/plugins/volumebinding/BUILD‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@ go_library(
1111
"//pkg/scheduler/apis/config:go_default_library",
1212
"//pkg/scheduler/framework/v1alpha1:go_default_library",
1313
"//staging/src/k8s.io/api/core/v1:go_default_library",
14+
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
1415
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
1516
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
17+
"//staging/src/k8s.io/client-go/listers/core/v1:go_default_library",
1618
"//vendor/k8s.io/klog/v2:go_default_library",
1719
],
1820
)

‎pkg/scheduler/framework/plugins/volumebinding/volume_binding.go‎

Lines changed: 38 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@ import (
2323
"time"
2424

2525
v1 "k8s.io/api/core/v1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2627
"k8s.io/apimachinery/pkg/runtime"
2728
utilfeature "k8s.io/apiserver/pkg/util/feature"
29+
corelisters "k8s.io/client-go/listers/core/v1"
2830
"k8s.io/klog/v2"
2931
"k8s.io/kubernetes/pkg/controller/volume/scheduling"
3032
"k8s.io/kubernetes/pkg/features"
@@ -62,6 +64,7 @@ func (d *stateData) Clone() framework.StateData {
6264
// Reserve and PreBind phases.
6365
type VolumeBinding struct {
6466
Binder scheduling.SchedulerVolumeBinder
67+
PVCLister corelisters.PersistentVolumeClaimLister
6568
GenericEphemeralVolumeFeatureEnabled bool
6669
}
6770

@@ -78,22 +81,50 @@ func (pl *VolumeBinding) Name() string {
7881
return Name
7982
}
8083

81-
func (pl *VolumeBinding) podHasPVCs(pod *v1.Pod) bool {
84+
// podHasPVCs returns 2 values:
85+
// - the first one to denote if the given "pod" has any PVC defined.
86+
// - the second one to return any error if the requested PVC is illegal.
87+
func (pl *VolumeBinding) podHasPVCs(pod *v1.Pod) (bool, error) {
88+
hasPVC := false
8289
for _, vol := range pod.Spec.Volumes {
83-
if vol.PersistentVolumeClaim != nil ||
84-
pl.GenericEphemeralVolumeFeatureEnabled && vol.Ephemeral != nil {
85-
return true
90+
var pvcName string
91+
ephemeral := false
92+
switch {
93+
case vol.PersistentVolumeClaim != nil:
94+
pvcName = vol.PersistentVolumeClaim.ClaimName
95+
case vol.Ephemeral != nil && pl.GenericEphemeralVolumeFeatureEnabled:
96+
pvcName = pod.Name + "-" + vol.Name
97+
ephemeral = true
98+
default:
99+
// Volume is not using a PVC, ignore
100+
continue
101+
}
102+
hasPVC = true
103+
pvc, err := pl.PVCLister.PersistentVolumeClaims(pod.Namespace).Get(pvcName)
104+
if err != nil {
105+
// The error has already enough context ("persistentvolumeclaim "myclaim" not found")
106+
return hasPVC, err
107+
}
108+
109+
if pvc.DeletionTimestamp != nil {
110+
return hasPVC, fmt.Errorf("persistentvolumeclaim %q is being deleted", pvc.Name)
111+
}
112+
113+
if ephemeral && !metav1.IsControlledBy(pvc, pod) {
114+
return hasPVC, fmt.Errorf("persistentvolumeclaim %q was not created for the pod", pvc.Name)
86115
}
87116
}
88-
return false
117+
return hasPVC, nil
89118
}
90119

91120
// PreFilter invoked at the prefilter extension point to check if pod has all
92121
// immediate PVCs bound. If not all immediate PVCs are bound, an
93122
// UnschedulableAndUnresolvable is returned.
94123
func (pl *VolumeBinding) PreFilter(ctx context.Context, state *framework.CycleState, pod *v1.Pod) *framework.Status {
95124
// If pod does not reference any PVC, we don't need to do anything.
96-
if !pl.podHasPVCs(pod) {
125+
if hasPVC, err := pl.podHasPVCs(pod); err != nil {
126+
return framework.NewStatus(framework.UnschedulableAndUnresolvable, err.Error())
127+
} else if !hasPVC {
97128
state.Write(stateKey, &stateData{skip: true})
98129
return nil
99130
}
@@ -271,6 +302,7 @@ func New(plArgs runtime.Object, fh framework.FrameworkHandle) (framework.Plugin,
271302
binder := scheduling.NewVolumeBinder(fh.ClientSet(), podInformer, nodeInformer, csiNodeInformer, pvcInformer, pvInformer, storageClassInformer, capacityCheck, time.Duration(args.BindTimeoutSeconds)*time.Second)
272303
return &VolumeBinding{
273304
Binder: binder,
305+
PVCLister: pvcInformer.Lister(),
274306
GenericEphemeralVolumeFeatureEnabled: utilfeature.DefaultFeatureGate.Enabled(features.GenericEphemeralVolume),
275307
}, nil
276308
}

‎pkg/scheduler/framework/plugins/volumebinding/volume_binding_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ func TestVolumeBinding(t *testing.T) {
209209
name: "pvc not found",
210210
pod: makePod("pod-a", []string{"pvc-a"}),
211211
node: &v1.Node{},
212-
wantPreFilterStatus: framework.NewStatus(framework.Error, `error getting PVC "default/pvc-a": could not find v1.PersistentVolumeClaim "default/pvc-a"`),
212+
wantPreFilterStatus: framework.NewStatus(framework.UnschedulableAndUnresolvable, `persistentvolumeclaim "pvc-a" not found`),
213213
wantFilterStatus: nil,
214214
},
215215
{

‎pkg/scheduler/scheduler_test.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -862,13 +862,14 @@ func setupTestSchedulerWithVolumeBinding(volumeBinder scheduling.SchedulerVolume
862862
testPVC := v1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "testPVC", Namespace: pod.Namespace, UID: types.UID("testPVC")}}
863863
client := clientsetfake.NewSimpleClientset(&testNode, &testPVC)
864864
informerFactory := informers.NewSharedInformerFactory(client, 0)
865-
informerFactory.Core().V1().PersistentVolumeClaims().Informer().GetStore().Add(&testPVC)
865+
pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims()
866+
pvcInformer.Informer().GetStore().Add(&testPVC)
866867

867868
fns := []st.RegisterPluginFunc{
868869
st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
869870
st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
870871
st.RegisterPluginAsExtensions(volumebinding.Name, func(plArgs runtime.Object, handle framework.FrameworkHandle) (framework.Plugin, error) {
871-
return &volumebinding.VolumeBinding{Binder: volumeBinder}, nil
872+
return &volumebinding.VolumeBinding{Binder: volumeBinder, PVCLister: pvcInformer.Lister()}, nil
872873
}, "PreFilter", "Filter", "Reserve", "PreBind"),
873874
}
874875
s, bindingChan, errChan := setupTestScheduler(queuedPodStore, scache, informerFactory, broadcaster, fns...)

0 commit comments

Comments
 (0)