Skip to content

Commit 2fdf1fa

Browse files
ahg-gahg-g
committed
inter-pod affinity prefilter
1 parent cfdc365 commit 2fdf1fa

File tree

10 files changed

+486
-2333
lines changed

10 files changed

+486
-2333
lines changed

‎pkg/scheduler/algorithm/predicates/metadata.go‎

Lines changed: 10 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,8 @@ func (m *serviceAffinityMetadata) clone() *serviceAffinityMetadata {
179179
return &copy
180180
}
181181

182-
type podAffinityMetadata struct {
182+
// PodAffinityMetadata pre-computed state for inter-pod affinity predicate.
183+
type PodAffinityMetadata struct {
183184
// A map of topology pairs to the number of existing pods that has anti-affinity terms that match the "pod".
184185
topologyToMatchedExistingAntiAffinityTerms topologyToMatchedTermCount
185186
// A map of topology pairs to the number of existing pods that match the affinity terms of the "pod".
@@ -225,7 +226,8 @@ func (m topologyToMatchedTermCount) updateWithAntiAffinityTerms(targetPod *v1.Po
225226
}
226227
}
227228

228-
func (m *podAffinityMetadata) updatePod(updatedPod, pod *v1.Pod, node *v1.Node, multiplier int64) error {
229+
// UpdateWithPod updates the metadata counters with the (anti)affinity matches for the given pod.
230+
func (m *PodAffinityMetadata) UpdateWithPod(updatedPod, pod *v1.Pod, node *v1.Node, multiplier int64) error {
229231
if m == nil {
230232
return nil
231233
}
@@ -265,12 +267,13 @@ func (m *podAffinityMetadata) updatePod(updatedPod, pod *v1.Pod, node *v1.Node,
265267
return nil
266268
}
267269

268-
func (m *podAffinityMetadata) clone() *podAffinityMetadata {
270+
// Clone makes a deep copy of PodAffinityMetadata.
271+
func (m *PodAffinityMetadata) Clone() *PodAffinityMetadata {
269272
if m == nil {
270273
return nil
271274
}
272275

273-
copy := podAffinityMetadata{}
276+
copy := PodAffinityMetadata{}
274277
copy.topologyToMatchedAffinityTerms = m.topologyToMatchedAffinityTerms.clone()
275278
copy.topologyToMatchedAntiAffinityTerms = m.topologyToMatchedAntiAffinityTerms.clone()
276279
copy.topologyToMatchedExistingAntiAffinityTerms = m.topologyToMatchedExistingAntiAffinityTerms.clone()
@@ -327,7 +330,6 @@ type predicateMetadata struct {
327330
evenPodsSpreadMetadata *evenPodsSpreadMetadata
328331

329332
serviceAffinityMetadata *serviceAffinityMetadata
330-
podAffinityMetadata *podAffinityMetadata
331333
podFitsResourcesMetadata *podFitsResourcesMetadata
332334
podFitsHostPortsMetadata *podFitsHostPortsMetadata
333335
}
@@ -373,20 +375,13 @@ func (f *MetadataProducerFactory) GetPredicateMetadata(pod *v1.Pod, sharedLister
373375
}
374376

375377
var allNodes []*schedulernodeinfo.NodeInfo
376-
var havePodsWithAffinityNodes []*schedulernodeinfo.NodeInfo
377378
if sharedLister != nil {
378379
var err error
379380
allNodes, err = sharedLister.NodeInfos().List()
380381
if err != nil {
381382
klog.Errorf("failed to list NodeInfos: %v", err)
382383
return nil
383384
}
384-
havePodsWithAffinityNodes, err = sharedLister.NodeInfos().HavePodsWithAffinityList()
385-
if err != nil {
386-
klog.Errorf("failed to list NodeInfos: %v", err)
387-
return nil
388-
}
389-
390385
}
391386

392387
// evenPodsSpreadMetadata represents how existing pods match "pod"
@@ -397,16 +392,9 @@ func (f *MetadataProducerFactory) GetPredicateMetadata(pod *v1.Pod, sharedLister
397392
return nil
398393
}
399394

400-
podAffinityMetadata, err := getPodAffinityMetadata(pod, allNodes, havePodsWithAffinityNodes)
401-
if err != nil {
402-
klog.Errorf("Error calculating podAffinityMetadata: %v", err)
403-
return nil
404-
}
405-
406395
predicateMetadata := &predicateMetadata{
407396
pod: pod,
408397
evenPodsSpreadMetadata: evenPodsSpreadMetadata,
409-
podAffinityMetadata: podAffinityMetadata,
410398
podFitsResourcesMetadata: getPodFitsResourcesMetedata(pod),
411399
podFitsHostPortsMetadata: getPodFitsHostPortsMetadata(pod),
412400
}
@@ -429,7 +417,8 @@ func getPodFitsResourcesMetedata(pod *v1.Pod) *podFitsResourcesMetadata {
429417
}
430418
}
431419

432-
func getPodAffinityMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo, havePodsWithAffinityNodes []*schedulernodeinfo.NodeInfo) (*podAffinityMetadata, error) {
420+
// GetPodAffinityMetadata computes inter-pod affinity metadata.
421+
func GetPodAffinityMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo, havePodsWithAffinityNodes []*schedulernodeinfo.NodeInfo) (*PodAffinityMetadata, error) {
433422
// existingPodAntiAffinityMap will be used later for efficient check on existing pods' anti-affinity
434423
existingPodAntiAffinityMap, err := getTPMapMatchingExistingAntiAffinity(pod, havePodsWithAffinityNodes)
435424
if err != nil {
@@ -442,7 +431,7 @@ func getPodAffinityMetadata(pod *v1.Pod, allNodes []*schedulernodeinfo.NodeInfo,
442431
return nil, err
443432
}
444433

445-
return &podAffinityMetadata{
434+
return &PodAffinityMetadata{
446435
topologyToMatchedAffinityTerms: incomingPodAffinityMap,
447436
topologyToMatchedAntiAffinityTerms: incomingPodAntiAffinityMap,
448437
topologyToMatchedExistingAntiAffinityTerms: existingPodAntiAffinityMap,
@@ -619,7 +608,6 @@ func (meta *predicateMetadata) RemovePod(deletedPod *v1.Pod, node *v1.Node) erro
619608
if deletedPodFullName == schedutil.GetPodFullName(meta.pod) {
620609
return fmt.Errorf("deletedPod and meta.pod must not be the same")
621610
}
622-
meta.podAffinityMetadata.updatePod(deletedPod, meta.pod, node, -1)
623611
meta.evenPodsSpreadMetadata.removePod(deletedPod, meta.pod, node)
624612
meta.serviceAffinityMetadata.removePod(deletedPod, node)
625613

@@ -637,9 +625,6 @@ func (meta *predicateMetadata) AddPod(addedPod *v1.Pod, node *v1.Node) error {
637625
return fmt.Errorf("node not found")
638626
}
639627

640-
if err := meta.podAffinityMetadata.updatePod(addedPod, meta.pod, node, 1); err != nil {
641-
return err
642-
}
643628
// Update meta.evenPodsSpreadMetadata if meta.pod has hard spread constraints
644629
// and addedPod matches that
645630
meta.evenPodsSpreadMetadata.addPod(addedPod, meta.pod, node)
@@ -657,7 +642,6 @@ func (meta *predicateMetadata) ShallowCopy() Metadata {
657642
podBestEffort: meta.podBestEffort,
658643
}
659644
newPredMeta.podFitsHostPortsMetadata = meta.podFitsHostPortsMetadata.clone()
660-
newPredMeta.podAffinityMetadata = meta.podAffinityMetadata.clone()
661645
newPredMeta.evenPodsSpreadMetadata = meta.evenPodsSpreadMetadata.clone()
662646
newPredMeta.serviceAffinityMetadata = meta.serviceAffinityMetadata.clone()
663647
newPredMeta.podFitsResourcesMetadata = meta.podFitsResourcesMetadata.clone()

‎pkg/scheduler/algorithm/predicates/metadata_test.go‎

Lines changed: 26 additions & 198 deletions
Original file line numberDiff line numberDiff line change
@@ -70,16 +70,7 @@ func predicateMetadataEquivalent(meta1, meta2 *predicateMetadata) error {
7070
for !reflect.DeepEqual(meta1.podFitsHostPortsMetadata.podPorts, meta2.podFitsHostPortsMetadata.podPorts) {
7171
return fmt.Errorf("podPorts are not equal")
7272
}
73-
if !reflect.DeepEqual(meta1.podAffinityMetadata.topologyToMatchedAffinityTerms, meta2.podAffinityMetadata.topologyToMatchedAffinityTerms) {
74-
return fmt.Errorf("topologyToMatchedAffinityTerms are not equal")
75-
}
76-
if !reflect.DeepEqual(meta1.podAffinityMetadata.topologyToMatchedAntiAffinityTerms, meta2.podAffinityMetadata.topologyToMatchedAntiAffinityTerms) {
77-
return fmt.Errorf("topologyToMatchedAntiAffinityTerms are not equal")
78-
}
79-
if !reflect.DeepEqual(meta1.podAffinityMetadata.topologyToMatchedExistingAntiAffinityTerms,
80-
meta2.podAffinityMetadata.topologyToMatchedExistingAntiAffinityTerms) {
81-
return fmt.Errorf("topologyToMatchedExistingAntiAffinityTerms are not equal, got: %v, want: %v", meta1.podAffinityMetadata.topologyToMatchedExistingAntiAffinityTerms, meta2.podAffinityMetadata.topologyToMatchedExistingAntiAffinityTerms)
82-
}
73+
8374
if meta1.serviceAffinityMetadata != nil {
8475
sortablePods1 := sortablePods(meta1.serviceAffinityMetadata.matchingPodList)
8576
sort.Sort(sortablePods1)
@@ -114,78 +105,6 @@ func TestPredicateMetadata_AddRemovePod(t *testing.T) {
114105
"zone": "z21",
115106
}
116107
selector1 := map[string]string{"foo": "bar"}
117-
antiAffinityFooBar := &v1.PodAntiAffinity{
118-
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
119-
{
120-
LabelSelector: &metav1.LabelSelector{
121-
MatchExpressions: []metav1.LabelSelectorRequirement{
122-
{
123-
Key: "foo",
124-
Operator: metav1.LabelSelectorOpIn,
125-
Values: []string{"bar"},
126-
},
127-
},
128-
},
129-
TopologyKey: "region",
130-
},
131-
},
132-
}
133-
antiAffinityComplex := &v1.PodAntiAffinity{
134-
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
135-
{
136-
LabelSelector: &metav1.LabelSelector{
137-
MatchExpressions: []metav1.LabelSelectorRequirement{
138-
{
139-
Key: "foo",
140-
Operator: metav1.LabelSelectorOpIn,
141-
Values: []string{"bar", "buzz"},
142-
},
143-
},
144-
},
145-
TopologyKey: "region",
146-
},
147-
{
148-
LabelSelector: &metav1.LabelSelector{
149-
MatchExpressions: []metav1.LabelSelectorRequirement{
150-
{
151-
Key: "service",
152-
Operator: metav1.LabelSelectorOpNotIn,
153-
Values: []string{"bar", "security", "test"},
154-
},
155-
},
156-
},
157-
TopologyKey: "zone",
158-
},
159-
},
160-
}
161-
affinityComplex := &v1.PodAffinity{
162-
RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{
163-
{
164-
LabelSelector: &metav1.LabelSelector{
165-
MatchExpressions: []metav1.LabelSelectorRequirement{
166-
{
167-
Key: "foo",
168-
Operator: metav1.LabelSelectorOpIn,
169-
Values: []string{"bar", "buzz"},
170-
},
171-
},
172-
},
173-
TopologyKey: "region",
174-
},
175-
{
176-
LabelSelector: &metav1.LabelSelector{
177-
MatchExpressions: []metav1.LabelSelectorRequirement{
178-
{
179-
Key: "service",
180-
Operator: metav1.LabelSelectorOpNotIn,
181-
Values: []string{"bar", "security", "test"},
182-
},
183-
},
184-
},
185-
TopologyKey: "zone",
186-
},
187-
},
188-
}
189108

190109
tests := []struct {
191110
name string
@@ -218,39 +137,6 @@ func TestPredicateMetadata_AddRemovePod(t *testing.T) {
218137
{ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}},
219138
},
220139
},
221-
{
222-
name: "metadata anti-affinity terms are updated correctly after adding and removing a pod",
223-
pendingPod: &v1.Pod{
224-
ObjectMeta: metav1.ObjectMeta{Name: "pending", Labels: selector1},
225-
},
226-
existingPods: []*v1.Pod{
227-
{ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1},
228-
Spec: v1.PodSpec{NodeName: "nodeA"},
229-
},
230-
{ObjectMeta: metav1.ObjectMeta{Name: "p2"},
231-
Spec: v1.PodSpec{
232-
NodeName: "nodeC",
233-
Affinity: &v1.Affinity{
234-
PodAntiAffinity: antiAffinityFooBar,
235-
},
236-
},
237-
},
238-
},
239-
addedPod: &v1.Pod{
240-
ObjectMeta: metav1.ObjectMeta{Name: "addedPod", Labels: selector1},
241-
Spec: v1.PodSpec{
242-
NodeName: "nodeB",
243-
Affinity: &v1.Affinity{
244-
PodAntiAffinity: antiAffinityFooBar,
245-
},
246-
},
247-
},
248-
nodes: []*v1.Node{
249-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeA", Labels: label1}},
250-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}},
251-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}},
252-
},
253-
},
254140
{
255141
name: "metadata service-affinity data are updated correctly after adding and removing a pod",
256142
pendingPod: &v1.Pod{
@@ -275,75 +161,6 @@ func TestPredicateMetadata_AddRemovePod(t *testing.T) {
275161
{ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}},
276162
},
277163
},
278-
{
279-
name: "metadata anti-affinity terms and service affinity data are updated correctly after adding and removing a pod",
280-
pendingPod: &v1.Pod{
281-
ObjectMeta: metav1.ObjectMeta{Name: "pending", Labels: selector1},
282-
},
283-
existingPods: []*v1.Pod{
284-
{ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1},
285-
Spec: v1.PodSpec{NodeName: "nodeA"},
286-
},
287-
{ObjectMeta: metav1.ObjectMeta{Name: "p2"},
288-
Spec: v1.PodSpec{
289-
NodeName: "nodeC",
290-
Affinity: &v1.Affinity{
291-
PodAntiAffinity: antiAffinityFooBar,
292-
},
293-
},
294-
},
295-
},
296-
addedPod: &v1.Pod{
297-
ObjectMeta: metav1.ObjectMeta{Name: "addedPod", Labels: selector1},
298-
Spec: v1.PodSpec{
299-
NodeName: "nodeA",
300-
Affinity: &v1.Affinity{
301-
PodAntiAffinity: antiAffinityComplex,
302-
},
303-
},
304-
},
305-
services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: selector1}}},
306-
nodes: []*v1.Node{
307-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeA", Labels: label1}},
308-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}},
309-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}},
310-
},
311-
},
312-
{
313-
name: "metadata matching pod affinity and anti-affinity are updated correctly after adding and removing a pod",
314-
pendingPod: &v1.Pod{
315-
ObjectMeta: metav1.ObjectMeta{Name: "pending", Labels: selector1},
316-
},
317-
existingPods: []*v1.Pod{
318-
{ObjectMeta: metav1.ObjectMeta{Name: "p1", Labels: selector1},
319-
Spec: v1.PodSpec{NodeName: "nodeA"},
320-
},
321-
{ObjectMeta: metav1.ObjectMeta{Name: "p2"},
322-
Spec: v1.PodSpec{
323-
NodeName: "nodeC",
324-
Affinity: &v1.Affinity{
325-
PodAntiAffinity: antiAffinityFooBar,
326-
PodAffinity: affinityComplex,
327-
},
328-
},
329-
},
330-
},
331-
addedPod: &v1.Pod{
332-
ObjectMeta: metav1.ObjectMeta{Name: "addedPod", Labels: selector1},
333-
Spec: v1.PodSpec{
334-
NodeName: "nodeA",
335-
Affinity: &v1.Affinity{
336-
PodAntiAffinity: antiAffinityComplex,
337-
},
338-
},
339-
},
340-
services: []*v1.Service{{Spec: v1.ServiceSpec{Selector: selector1}}},
341-
nodes: []*v1.Node{
342-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeA", Labels: label1}},
343-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeB", Labels: label2}},
344-
{ObjectMeta: metav1.ObjectMeta{Name: "nodeC", Labels: label3}},
345-
},
346-
},
347164
}
348165

349166
for _, test := range tests {
@@ -385,6 +202,31 @@ func TestPredicateMetadata_AddRemovePod(t *testing.T) {
385202
}
386203
}
387204

205+
func TestPodAffinityMetadata_Clone(t *testing.T) {
206+
source := &PodAffinityMetadata{
207+
topologyToMatchedExistingAntiAffinityTerms: topologyToMatchedTermCount{
208+
{key: "name", value: "machine1"}: 1,
209+
{key: "name", value: "machine2"}: 1,
210+
},
211+
topologyToMatchedAffinityTerms: topologyToMatchedTermCount{
212+
{key: "name", value: "nodeA"}: 1,
213+
{key: "name", value: "nodeC"}: 2,
214+
},
215+
topologyToMatchedAntiAffinityTerms: topologyToMatchedTermCount{
216+
{key: "name", value: "nodeN"}: 3,
217+
{key: "name", value: "nodeM"}: 1,
218+
},
219+
}
220+
221+
clone := source.Clone()
222+
if clone == source {
223+
t.Errorf("Clone returned the exact same object!")
224+
}
225+
if !reflect.DeepEqual(clone, source) {
226+
t.Errorf("Copy is not equal to source!")
227+
}
228+
}
229+
388230
// TestPredicateMetadata_ShallowCopy tests the ShallowCopy function. It is based
389231
// on the idea that shallow-copy should produce an object that is deep-equal to the original
390232
// object.
@@ -415,20 +257,6 @@ func TestPredicateMetadata_ShallowCopy(t *testing.T) {
415257
},
416258
},
417259
},
418-
podAffinityMetadata: &podAffinityMetadata{
419-
topologyToMatchedExistingAntiAffinityTerms: topologyToMatchedTermCount{
420-
{key: "name", value: "machine1"}: 1,
421-
{key: "name", value: "machine2"}: 1,
422-
},
423-
topologyToMatchedAffinityTerms: topologyToMatchedTermCount{
424-
{key: "name", value: "nodeA"}: 1,
425-
{key: "name", value: "nodeC"}: 2,
426-
},
427-
topologyToMatchedAntiAffinityTerms: topologyToMatchedTermCount{
428-
{key: "name", value: "nodeN"}: 3,
429-
{key: "name", value: "nodeM"}: 1,
430-
},
431-
},
432260
evenPodsSpreadMetadata: &evenPodsSpreadMetadata{
433261
tpKeyToCriticalPaths: map[string]*criticalPaths{
434262
"name": {{"nodeA", 1}, {"nodeC", 2}},

0 commit comments

Comments
 (0)