Skip to content

Commit ded2ff3

Browse files
authored
Merge pull request kubernetes#86249 from ahg-g/ahg1-noderes1
PodFitsResources Predicate Metadata as prefilter
2 parents 5ead497 + 02a0aad commit ded2ff3

File tree

11 files changed

+170
-168
lines changed

11 files changed

+170
-168
lines changed

pkg/scheduler/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ go_library(
2222
"//pkg/scheduler/core:go_default_library",
2323
"//pkg/scheduler/framework/plugins:go_default_library",
2424
"//pkg/scheduler/framework/plugins/nodelabel:go_default_library",
25+
"//pkg/scheduler/framework/plugins/noderesources:go_default_library",
2526
"//pkg/scheduler/framework/plugins/requestedtocapacityratio:go_default_library",
2627
"//pkg/scheduler/framework/plugins/serviceaffinity:go_default_library",
2728
"//pkg/scheduler/framework/v1alpha1:go_default_library",

pkg/scheduler/algorithm/predicates/metadata.go

Lines changed: 2 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -281,36 +281,12 @@ func (m *PodAffinityMetadata) Clone() *PodAffinityMetadata {
281281
return &copy
282282
}
283283

284-
type podFitsResourcesMetadata struct {
285-
// ignoredExtendedResources is a set of extended resource names that will
286-
// be ignored in the PodFitsResources predicate.
287-
//
288-
// They can be scheduler extender managed resources, the consumption of
289-
// which should be accounted only by the extenders. This set is synthesized
290-
// from scheduler extender configuration and does not change per pod.
291-
ignoredExtendedResources sets.String
292-
podRequest *schedulernodeinfo.Resource
293-
}
294-
295-
func (m *podFitsResourcesMetadata) clone() *podFitsResourcesMetadata {
296-
if m == nil {
297-
return nil
298-
}
299-
300-
copy := podFitsResourcesMetadata{}
301-
copy.ignoredExtendedResources = m.ignoredExtendedResources
302-
copy.podRequest = m.podRequest
303-
304-
return &copy
305-
}
306-
307284
// NOTE: When new fields are added/removed or logic is changed, please make sure that
308285
// RemovePod, AddPod, and ShallowCopy functions are updated to work with the new changes.
309286
type predicateMetadata struct {
310287
pod *v1.Pod
311288

312-
serviceAffinityMetadata *serviceAffinityMetadata
313-
podFitsResourcesMetadata *podFitsResourcesMetadata
289+
serviceAffinityMetadata *serviceAffinityMetadata
314290
}
315291

316292
// Ensure that predicateMetadata implements algorithm.Metadata.
@@ -332,17 +308,6 @@ func EmptyMetadataProducer(pod *v1.Pod, sharedLister schedulerlisters.SharedList
332308
return nil
333309
}
334310

335-
// RegisterPredicateMetadataProducerWithExtendedResourceOptions registers a
336-
// MetadataProducer that creates predicate metadata with the provided
337-
// options for extended resources.
338-
//
339-
// See the comments in "predicateMetadata" for the explanation of the options.
340-
func RegisterPredicateMetadataProducerWithExtendedResourceOptions(ignoredExtendedResources sets.String) {
341-
RegisterPredicateMetadataProducer("PredicateWithExtendedResourceOptions", func(pm *predicateMetadata) {
342-
pm.podFitsResourcesMetadata.ignoredExtendedResources = ignoredExtendedResources
343-
})
344-
}
345-
346311
// MetadataProducerFactory is a factory to produce Metadata.
347312
type MetadataProducerFactory struct{}
348313

@@ -354,8 +319,7 @@ func (f *MetadataProducerFactory) GetPredicateMetadata(pod *v1.Pod, sharedLister
354319
}
355320

356321
predicateMetadata := &predicateMetadata{
357-
pod: pod,
358-
podFitsResourcesMetadata: getPodFitsResourcesMetedata(pod),
322+
pod: pod,
359323
}
360324
for predicateName, precomputeFunc := range predicateMetadataProducers {
361325
klog.V(10).Infof("Precompute: %v", predicateName)
@@ -364,12 +328,6 @@ func (f *MetadataProducerFactory) GetPredicateMetadata(pod *v1.Pod, sharedLister
364328
return predicateMetadata
365329
}
366330

367-
func getPodFitsResourcesMetedata(pod *v1.Pod) *podFitsResourcesMetadata {
368-
return &podFitsResourcesMetadata{
369-
podRequest: GetResourceRequest(pod),
370-
}
371-
}
372-
373331
// GetPodAffinityMetadata computes inter-pod affinity metadata.
374332
func GetPodAffinityMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo, havePodsWithAffinityNodes []*schedulernodeinfo.NodeInfo) (*PodAffinityMetadata, error) {
375333
// existingPodAntiAffinityMap will be used later for efficient check on existing pods' anti-affinity
@@ -593,7 +551,6 @@ func (meta *predicateMetadata) ShallowCopy() Metadata {
593551
pod: meta.pod,
594552
}
595553
newPredMeta.serviceAffinityMetadata = meta.serviceAffinityMetadata.clone()
596-
newPredMeta.podFitsResourcesMetadata = meta.podFitsResourcesMetadata.clone()
597554
return (Metadata)(newPredMeta)
598555
}
599556

pkg/scheduler/algorithm/predicates/metadata_test.go

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -228,13 +228,6 @@ func TestPredicateMetadata_ShallowCopy(t *testing.T) {
228228
Namespace: "testns",
229229
},
230230
},
231-
podFitsResourcesMetadata: &podFitsResourcesMetadata{
232-
podRequest: &schedulernodeinfo.Resource{
233-
MilliCPU: 1000,
234-
Memory: 300,
235-
AllowedPodNumber: 4,
236-
},
237-
},
238231
serviceAffinityMetadata: &serviceAffinityMetadata{
239232
matchingPodList: []*v1.Pod{
240233
{ObjectMeta: metav1.ObjectMeta{Name: "pod1"}},

pkg/scheduler/algorithm/predicates/predicates.go

Lines changed: 21 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -780,10 +780,16 @@ func podName(pod *v1.Pod) string {
780780
return pod.Namespace + "/" + pod.Name
781781
}
782782

783-
// PodFitsResources checks if a node has sufficient resources, such as cpu, memory, gpu, opaque int resources etc to run a pod.
783+
// PodFitsResources is a wrapper around PodFitsResourcesPredicate that implements FitPredicate interface.
784+
// TODO(#85822): remove this function once predicate registration logic is deleted.
785+
func PodFitsResources(pod *v1.Pod, _ Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
786+
return PodFitsResourcesPredicate(pod, nil, nil, nodeInfo)
787+
}
788+
789+
// PodFitsResourcesPredicate checks if a node has sufficient resources, such as cpu, memory, gpu, opaque int resources etc to run a pod.
784790
// First return value indicates whether a node has sufficient resources to run a pod while the second return value indicates the
785-
// predicate failure reasons if the node has insufficient resources to run the pod.
786-
func PodFitsResources(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
791+
// predicate failure reasons if the node has insufficient resources to run the pod
792+
func PodFitsResourcesPredicate(pod *v1.Pod, podRequest *schedulernodeinfo.Resource, ignoredExtendedResources sets.String, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
787793
node := nodeInfo.Node()
788794
if node == nil {
789795
return false, nil, fmt.Errorf("node not found")
@@ -795,17 +801,11 @@ func PodFitsResources(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.No
795801
predicateFails = append(predicateFails, NewInsufficientResourceError(v1.ResourcePods, 1, int64(len(nodeInfo.Pods())), int64(allowedPodNumber)))
796802
}
797803

798-
// No extended resources should be ignored by default.
799-
ignoredExtendedResources := sets.NewString()
804+
if ignoredExtendedResources == nil {
805+
ignoredExtendedResources = sets.NewString()
806+
}
800807

801-
var podRequest *schedulernodeinfo.Resource
802-
if predicateMeta, ok := meta.(*predicateMetadata); ok && predicateMeta.podFitsResourcesMetadata != nil {
803-
podRequest = predicateMeta.podFitsResourcesMetadata.podRequest
804-
if predicateMeta.podFitsResourcesMetadata.ignoredExtendedResources != nil {
805-
ignoredExtendedResources = predicateMeta.podFitsResourcesMetadata.ignoredExtendedResources
806-
}
807-
} else {
808-
// We couldn't parse metadata - fallback to computing it.
808+
if podRequest == nil {
809809
podRequest = GetResourceRequest(pod)
810810
}
811811
if podRequest.MilliCPU == 0 &&
@@ -839,13 +839,11 @@ func PodFitsResources(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.No
839839
}
840840
}
841841

842-
if klog.V(10) {
843-
if len(predicateFails) == 0 {
844-
// We explicitly don't do klog.V(10).Infof() to avoid computing all the parameters if this is
845-
// not logged. There is visible performance gain from it.
846-
klog.Infof("Schedule Pod %+v on Node %+v is allowed, Node is running only %v out of %v Pods.",
847-
podName(pod), node.Name, len(nodeInfo.Pods()), allowedPodNumber)
848-
}
842+
if klog.V(10) && len(predicateFails) == 0 {
843+
// We explicitly don't do klog.V(10).Infof() to avoid computing all the parameters if this is
844+
// not logged. There is visible performance gain from it.
845+
klog.Infof("Schedule Pod %+v on Node %+v is allowed, Node is running only %v out of %v Pods.",
846+
podName(pod), node.Name, len(nodeInfo.Pods()), allowedPodNumber)
849847
}
850848
return len(predicateFails) == 0, predicateFails, nil
851849
}
@@ -1144,43 +1142,11 @@ func haveOverlap(a1, a2 []string) bool {
11441142
return false
11451143
}
11461144

1147-
// GeneralPredicates checks whether noncriticalPredicates and EssentialPredicates pass. noncriticalPredicates are the predicates
1148-
// that only non-critical pods need and EssentialPredicates are the predicates that all pods, including critical pods, need.
1145+
// GeneralPredicates checks a group of predicates that the kubelet cares about.
1146+
// DEPRECATED: this exist only because kubelet uses it. We should change kubelet to execute the individual predicates it requires.
11491147
func GeneralPredicates(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
11501148
var predicateFails []PredicateFailureReason
1151-
for _, predicate := range []FitPredicate{noncriticalPredicates, EssentialPredicates} {
1152-
fit, reasons, err := predicate(pod, meta, nodeInfo)
1153-
if err != nil {
1154-
return false, predicateFails, err
1155-
}
1156-
if !fit {
1157-
predicateFails = append(predicateFails, reasons...)
1158-
}
1159-
}
1160-
1161-
return len(predicateFails) == 0, predicateFails, nil
1162-
}
1163-
1164-
// noncriticalPredicates are the predicates that only non-critical pods need.
1165-
func noncriticalPredicates(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
1166-
var predicateFails []PredicateFailureReason
1167-
fit, reasons, err := PodFitsResources(pod, meta, nodeInfo)
1168-
if err != nil {
1169-
return false, predicateFails, err
1170-
}
1171-
if !fit {
1172-
predicateFails = append(predicateFails, reasons...)
1173-
}
1174-
1175-
return len(predicateFails) == 0, predicateFails, nil
1176-
}
1177-
1178-
// EssentialPredicates are the predicates that all pods, including critical pods, need.
1179-
func EssentialPredicates(pod *v1.Pod, meta Metadata, nodeInfo *schedulernodeinfo.NodeInfo) (bool, []PredicateFailureReason, error) {
1180-
var predicateFails []PredicateFailureReason
1181-
// TODO: PodFitsHostPorts is essential for now, but kubelet should ideally
1182-
// preempt pods to free up host ports too
1183-
for _, predicate := range []FitPredicate{PodFitsHost, PodFitsHostPorts, PodMatchNodeSelector} {
1149+
for _, predicate := range []FitPredicate{PodFitsResources, PodFitsHost, PodFitsHostPorts, PodMatchNodeSelector} {
11841150
fit, reasons, err := predicate(pod, meta, nodeInfo)
11851151
if err != nil {
11861152
return false, predicateFails, err

pkg/scheduler/algorithm/predicates/predicates_test.go

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -388,10 +388,7 @@ func TestPodFitsResources(t *testing.T) {
388388
t.Run(test.name, func(t *testing.T) {
389389
node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 32, 5, 20, 5).Capacity, Allocatable: makeAllocatableResources(10, 20, 32, 5, 20, 5)}}
390390
test.nodeInfo.SetNode(&node)
391-
RegisterPredicateMetadataProducerWithExtendedResourceOptions(test.ignoredExtendedResources)
392-
factory := &MetadataProducerFactory{}
393-
meta := factory.GetPredicateMetadata(test.pod, nil)
394-
fits, reasons, err := PodFitsResources(test.pod, meta, test.nodeInfo)
391+
fits, reasons, err := PodFitsResourcesPredicate(test.pod, GetResourceRequest(test.pod), test.ignoredExtendedResources, test.nodeInfo)
395392
if err != nil {
396393
t.Errorf("unexpected error: %v", err)
397394
}
@@ -448,8 +445,8 @@ func TestPodFitsResources(t *testing.T) {
448445
t.Run(test.name, func(t *testing.T) {
449446
node := v1.Node{Status: v1.NodeStatus{Capacity: v1.ResourceList{}, Allocatable: makeAllocatableResources(10, 20, 1, 0, 0, 0)}}
450447
test.nodeInfo.SetNode(&node)
451-
factory := &MetadataProducerFactory{}
452-
fits, reasons, err := PodFitsResources(test.pod, factory.GetPredicateMetadata(test.pod, nil), test.nodeInfo)
448+
fits, reasons, err := PodFitsResourcesPredicate(test.pod, GetResourceRequest(test.pod), nil, test.nodeInfo)
449+
453450
if err != nil {
454451
t.Errorf("unexpected error: %v", err)
455452
}
@@ -509,8 +506,7 @@ func TestPodFitsResources(t *testing.T) {
509506
t.Run(test.name, func(t *testing.T) {
510507
node := v1.Node{Status: v1.NodeStatus{Capacity: makeResources(10, 20, 32, 5, 20, 5).Capacity, Allocatable: makeAllocatableResources(10, 20, 32, 5, 20, 5)}}
511508
test.nodeInfo.SetNode(&node)
512-
factory := &MetadataProducerFactory{}
513-
fits, reasons, err := PodFitsResources(test.pod, factory.GetPredicateMetadata(test.pod, nil), test.nodeInfo)
509+
fits, reasons, err := PodFitsResourcesPredicate(test.pod, GetResourceRequest(test.pod), nil, test.nodeInfo)
514510
if err != nil {
515511
t.Errorf("unexpected error: %v", err)
516512
}

pkg/scheduler/factory.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import (
4444
"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
4545
"k8s.io/kubernetes/pkg/scheduler/core"
4646
"k8s.io/kubernetes/pkg/scheduler/framework/plugins"
47+
"k8s.io/kubernetes/pkg/scheduler/framework/plugins/noderesources"
4748
framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
4849
internalcache "k8s.io/kubernetes/pkg/scheduler/internal/cache"
4950
cachedebugger "k8s.io/kubernetes/pkg/scheduler/internal/cache/debugger"
@@ -177,8 +178,8 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler,
177178

178179
var extenders []algorithm.SchedulerExtender
179180
if len(policy.Extenders) != 0 {
180-
ignoredExtendedResources := sets.NewString()
181181
var ignorableExtenders []algorithm.SchedulerExtender
182+
var ignoredExtendedResources []string
182183
for ii := range policy.Extenders {
183184
klog.V(2).Infof("Creating extender with config %+v", policy.Extenders[ii])
184185
extender, err := core.NewHTTPExtender(&policy.Extenders[ii])
@@ -192,13 +193,15 @@ func (c *Configurator) CreateFromConfig(policy schedulerapi.Policy) (*Scheduler,
192193
}
193194
for _, r := range policy.Extenders[ii].ManagedResources {
194195
if r.IgnoredByScheduler {
195-
ignoredExtendedResources.Insert(string(r.Name))
196+
ignoredExtendedResources = append(ignoredExtendedResources, r.Name)
196197
}
197198
}
198199
}
200+
c.configProducerArgs.NodeResourcesFitArgs = &noderesources.FitArgs{
201+
IgnoredResources: ignoredExtendedResources,
202+
}
199203
// place ignorable extenders to the tail of extenders
200204
extenders = append(extenders, ignorableExtenders...)
201-
predicates.RegisterPredicateMetadataProducerWithExtendedResourceOptions(ignoredExtendedResources)
202205
}
203206
// Providing HardPodAffinitySymmetricWeight in the policy config is the new and preferred way of providing the value.
204207
// Give it higher precedence than scheduler CLI configuration when it is provided.

pkg/scheduler/framework/plugins/default_registry.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,8 @@ type ConfigProducerArgs struct {
9999
RequestedToCapacityRatioArgs *requestedtocapacityratio.Args
100100
// ServiceAffinityArgs is the args for the ServiceAffinity plugin.
101101
ServiceAffinityArgs *serviceaffinity.Args
102+
// NodeResourcesFitArgs is the args for the NodeResources fit filter.
103+
NodeResourcesFitArgs *noderesources.FitArgs
102104
}
103105

104106
// ConfigProducer produces a framework's configuration.
@@ -119,9 +121,10 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
119121
}
120122
// Register Predicates.
121123
registry.RegisterPredicate(predicates.GeneralPred,
122-
func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
124+
func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
123125
// GeneralPredicate is a combination of predicates.
124126
plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil)
127+
pluginConfig = append(pluginConfig, makePluginConfig(noderesources.FitName, args.NodeResourcesFitArgs))
125128
plugins.Filter = appendToPluginSet(plugins.Filter, nodename.Name, nil)
126129
plugins.Filter = appendToPluginSet(plugins.Filter, nodeports.Name, nil)
127130
plugins.Filter = appendToPluginSet(plugins.Filter, nodeaffinity.Name, nil)
@@ -133,8 +136,9 @@ func NewDefaultConfigProducerRegistry() *ConfigProducerRegistry {
133136
return
134137
})
135138
registry.RegisterPredicate(predicates.PodFitsResourcesPred,
136-
func(_ ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
139+
func(args ConfigProducerArgs) (plugins config.Plugins, pluginConfig []config.PluginConfig) {
137140
plugins.Filter = appendToPluginSet(plugins.Filter, noderesources.FitName, nil)
141+
pluginConfig = append(pluginConfig, makePluginConfig(noderesources.FitName, args.NodeResourcesFitArgs))
138142
return
139143
})
140144
registry.RegisterPredicate(predicates.HostNamePred,

pkg/scheduler/framework/plugins/noderesources/BUILD

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ go_library(
2121
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
2222
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
2323
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
24+
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
25+
"//vendor/k8s.io/klog:go_default_library",
2426
],
2527
)
2628

@@ -51,14 +53,13 @@ go_test(
5153
"//pkg/apis/core/v1/helper:go_default_library",
5254
"//pkg/features:go_default_library",
5355
"//pkg/scheduler/algorithm/predicates:go_default_library",
54-
"//pkg/scheduler/framework/plugins/migration:go_default_library",
5556
"//pkg/scheduler/framework/v1alpha1:go_default_library",
5657
"//pkg/scheduler/nodeinfo:go_default_library",
5758
"//pkg/scheduler/nodeinfo/snapshot:go_default_library",
5859
"//staging/src/k8s.io/api/core/v1:go_default_library",
5960
"//staging/src/k8s.io/apimachinery/pkg/api/resource:go_default_library",
6061
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
61-
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
62+
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
6263
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
6364
"//staging/src/k8s.io/component-base/featuregate/testing:go_default_library",
6465
],

0 commit comments

Comments
 (0)