Skip to content

Commit a8caae5

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#86028 from ahg-g/ahg1-benchmarks
Fix inter-pod affinity scheduler benchmarks
2 parents 1fd2137 + a051c59 commit a8caae5

File tree

1 file changed

+62
-31
lines changed

1 file changed

+62
-31
lines changed

‎test/integration/scheduler_perf/scheduler_bench_test.go‎

Lines changed: 62 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -45,20 +45,23 @@ var (
4545
// From PV controller
4646
annBindCompleted = "pv.kubernetes.io/bind-completed"
4747

48-
tests = []struct{ nodes, existingPods, minPods int }{
48+
defaultTests = []struct{ nodes, existingPods, minPods int }{
4949
{nodes: 500, existingPods: 500, minPods: 1000},
5050
{nodes: 5000, existingPods: 5000, minPods: 1000},
5151
}
52+
testNamespace = "sched-test"
53+
setupNamespace = "sched-setup"
5254
)
5355

5456
// BenchmarkScheduling benchmarks the scheduling rate when the cluster has
5557
// various quantities of nodes and scheduled pods.
5658
func BenchmarkScheduling(b *testing.B) {
5759
testStrategy := testutils.NewSimpleWithControllerCreatePodStrategy("rc1")
58-
for _, test := range tests {
60+
for _, test := range defaultTests {
5961
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
6062
b.Run(name, func(b *testing.B) {
61-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
63+
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
64+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
6265
})
6366
}
6467
}
@@ -67,15 +70,26 @@ func BenchmarkScheduling(b *testing.B) {
6770
// PodAntiAffinity rules when the cluster has various quantities of nodes and
6871
// scheduled pods.
6972
func BenchmarkSchedulingPodAntiAffinity(b *testing.B) {
73+
// Since the pods has anti affinity to each other, the number of pods to schedule
74+
// can't exceed the number of nodes (the topology used in the test)
75+
tests := []struct{ nodes, existingPods, minPods int }{
76+
{nodes: 500, existingPods: 100, minPods: 400},
77+
{nodes: 5000, existingPods: 1000, minPods: 1000},
78+
}
7079
testBasePod := makeBasePodWithPodAntiAffinity(
7180
map[string]string{"name": "test", "color": "green"},
7281
map[string]string{"color": "green"})
73-
// The test strategy creates pods with anti-affinity for each other.
82+
// The test strategy creates pods with anti-affinity to each other, each pod ending up in a separate node.
7483
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
7584
for _, test := range tests {
7685
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
7786
b.Run(name, func(b *testing.B) {
78-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
87+
var nodeStrategies []testutils.CountToStrategy
88+
for i := 0; i < test.nodes; i++ {
89+
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
90+
nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
91+
}
92+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
7993
})
8094
}
8195
}
@@ -88,10 +102,11 @@ func BenchmarkSchedulingSecrets(b *testing.B) {
88102
// The test strategy creates pods with a secret.
89103
testBasePod := makeBasePodWithSecret()
90104
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
91-
for _, test := range tests {
105+
for _, test := range defaultTests {
92106
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
93107
b.Run(name, func(b *testing.B) {
94-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
108+
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
109+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
95110
})
96111
}
97112
}
@@ -104,10 +119,11 @@ func BenchmarkSchedulingInTreePVs(b *testing.B) {
104119
baseClaim := makeBasePersistentVolumeClaim()
105120
basePod := makeBasePod()
106121
testStrategy := testutils.NewCreatePodWithPersistentVolumeStrategy(baseClaim, awsVolumeFactory, basePod)
107-
for _, test := range tests {
122+
for _, test := range defaultTests {
108123
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
109124
b.Run(name, func(b *testing.B) {
110-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, defaultNodeStrategy, testStrategy, b)
125+
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: defaultNodeStrategy}}
126+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
111127
})
112128
}
113129
}
@@ -134,12 +150,13 @@ func BenchmarkSchedulingMigratedInTreePVs(b *testing.B) {
134150
},
135151
}
136152
nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{csilibplugins.AWSEBSInTreePluginName})
137-
for _, test := range tests {
153+
for _, test := range defaultTests {
138154
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
139155
b.Run(name, func(b *testing.B) {
140156
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigration, true)()
141157
defer featuregatetesting.SetFeatureGateDuringTest(b, utilfeature.DefaultFeatureGate, features.CSIMigrationAWS, true)()
142-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
158+
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
159+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
143160
})
144161
}
145162
}
@@ -164,10 +181,11 @@ func BenchmarkSchedulingCSIPVs(b *testing.B) {
164181
},
165182
}
166183
nodeStrategy := testutils.NewNodeAllocatableStrategy(allocatable, csiAllocatable, []string{})
167-
for _, test := range tests {
184+
for _, test := range defaultTests {
168185
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
169186
b.Run(name, func(b *testing.B) {
170-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
187+
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
188+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
171189
})
172190
}
173191
}
@@ -183,10 +201,11 @@ func BenchmarkSchedulingPodAffinity(b *testing.B) {
183201
// The test strategy creates pods with affinity for each other.
184202
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
185203
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
186-
for _, test := range tests {
204+
for _, test := range defaultTests {
187205
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
188206
b.Run(name, func(b *testing.B) {
189-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
207+
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
208+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
190209
})
191210
}
192211
}
@@ -201,11 +220,15 @@ func BenchmarkSchedulingPreferredPodAffinity(b *testing.B) {
201220
)
202221
// The test strategy creates pods with affinity for each other.
203222
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
204-
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
205-
for _, test := range tests {
223+
for _, test := range defaultTests {
206224
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
207225
b.Run(name, func(b *testing.B) {
208-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
226+
var nodeStrategies []testutils.CountToStrategy
227+
for i := 0; i < test.nodes; i++ {
228+
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
229+
nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
230+
}
231+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
209232
})
210233
}
211234
}
@@ -218,13 +241,17 @@ func BenchmarkSchedulingPreferredPodAntiAffinity(b *testing.B) {
218241
map[string]string{"foo": ""},
219242
map[string]string{"foo": ""},
220243
)
221-
// The test strategy creates pods with affinity for each other.
244+
// The test strategy creates pods with anti affinity to each other.
222245
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
223-
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
224-
for _, test := range tests {
246+
for _, test := range defaultTests {
225247
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
226248
b.Run(name, func(b *testing.B) {
227-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
249+
var nodeStrategies []testutils.CountToStrategy
250+
for i := 0; i < test.nodes; i++ {
251+
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelHostname, fmt.Sprintf("node-%d", i))
252+
nodeStrategies = append(nodeStrategies, testutils.CountToStrategy{Count: 1, Strategy: nodeStrategy})
253+
}
254+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
228255
})
229256
}
230257
}
@@ -237,10 +264,11 @@ func BenchmarkSchedulingNodeAffinity(b *testing.B) {
237264
// The test strategy creates pods with node-affinity for each other.
238265
testStrategy := testutils.NewCustomCreatePodStrategy(testBasePod)
239266
nodeStrategy := testutils.NewLabelNodePrepareStrategy(v1.LabelZoneFailureDomain, "zone1")
240-
for _, test := range tests {
267+
for _, test := range defaultTests {
241268
name := fmt.Sprintf("%vNodes/%vPods", test.nodes, test.existingPods)
242269
b.Run(name, func(b *testing.B) {
243-
benchmarkScheduling(test.nodes, test.existingPods, test.minPods, nodeStrategy, testStrategy, b)
270+
nodeStrategies := []testutils.CountToStrategy{{Count: test.nodes, Strategy: nodeStrategy}}
271+
benchmarkScheduling(test.existingPods, test.minPods, nodeStrategies, testStrategy, b)
244272
})
245273
}
246274
}
@@ -263,6 +291,7 @@ func makeBasePodWithPodAntiAffinity(podLabels, affinityLabels map[string]string)
263291
MatchLabels: affinityLabels,
264292
},
265293
TopologyKey: v1.LabelHostname,
294+
Namespaces: []string{testNamespace, setupNamespace},
266295
},
267296
},
268297
},
@@ -289,6 +318,7 @@ func makeBasePodWithPreferredPodAntiAffinity(podLabels, affinityLabels map[strin
289318
MatchLabels: affinityLabels,
290319
},
291320
TopologyKey: v1.LabelHostname,
321+
Namespaces: []string{testNamespace, setupNamespace},
292322
},
293323
Weight: 1,
294324
},
@@ -317,6 +347,7 @@ func makeBasePodWithPreferredPodAffinity(podLabels, affinityLabels map[string]st
317347
MatchLabels: affinityLabels,
318348
},
319349
TopologyKey: v1.LabelHostname,
350+
Namespaces: []string{testNamespace, setupNamespace},
320351
},
321352
Weight: 1,
322353
},
@@ -344,6 +375,7 @@ func makeBasePodWithPodAffinity(podLabels, affinityZoneLabels map[string]string)
344375
MatchLabels: affinityZoneLabels,
345376
},
346377
TopologyKey: v1.LabelZoneFailureDomain,
378+
Namespaces: []string{testNamespace, setupNamespace},
347379
},
348380
},
349381
},
@@ -384,8 +416,8 @@ func makeBasePodWithNodeAffinity(key string, vals []string) *v1.Pod {
384416
// and specific number of pods already scheduled.
385417
// This will schedule numExistingPods pods before the benchmark starts, and at
386418
// least minPods pods during the benchmark.
387-
func benchmarkScheduling(numNodes, numExistingPods, minPods int,
388-
nodeStrategy testutils.PrepareNodeStrategy,
419+
func benchmarkScheduling(numExistingPods, minPods int,
420+
nodeStrategies []testutils.CountToStrategy,
389421
testPodStrategy testutils.TestPodCreateStrategy,
390422
b *testing.B) {
391423
if b.N < minPods {
@@ -396,16 +428,15 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int,
396428

397429
nodePreparer := framework.NewIntegrationTestNodePreparer(
398430
clientset,
399-
[]testutils.CountToStrategy{{Count: numNodes, Strategy: nodeStrategy}},
400-
"scheduler-perf-",
401-
)
431+
nodeStrategies,
432+
"scheduler-perf-")
402433
if err := nodePreparer.PrepareNodes(); err != nil {
403434
klog.Fatalf("%v", err)
404435
}
405436
defer nodePreparer.CleanupNodes()
406437

407438
config := testutils.NewTestPodCreatorConfig()
408-
config.AddStrategy("sched-setup", numExistingPods, testPodStrategy)
439+
config.AddStrategy(setupNamespace, numExistingPods, testPodStrategy)
409440
podCreator := testutils.NewTestPodCreator(clientset, config)
410441
podCreator.CreatePods()
411442

@@ -439,7 +470,7 @@ func benchmarkScheduling(numNodes, numExistingPods, minPods int,
439470
// start benchmark
440471
b.ResetTimer()
441472
config = testutils.NewTestPodCreatorConfig()
442-
config.AddStrategy("sched-test", b.N, testPodStrategy)
473+
config.AddStrategy(testNamespace, b.N, testPodStrategy)
443474
podCreator = testutils.NewTestPodCreator(clientset, config)
444475
podCreator.CreatePods()
445476

0 commit comments

Comments
 (0)