@@ -26,6 +26,8 @@ import (
26
26
"testing"
27
27
"time"
28
28
29
+ "github.com/google/go-cmp/cmp"
30
+
29
31
v1 "k8s.io/api/core/v1"
30
32
"k8s.io/apimachinery/pkg/api/errors"
31
33
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -38,6 +40,7 @@ import (
38
40
clientset "k8s.io/client-go/kubernetes"
39
41
listersv1 "k8s.io/client-go/listers/core/v1"
40
42
featuregatetesting "k8s.io/component-base/featuregate/testing"
43
+ corev1helpers "k8s.io/component-helpers/scheduling/corev1"
41
44
configv1 "k8s.io/kube-scheduler/config/v1"
42
45
"k8s.io/kubernetes/pkg/features"
43
46
"k8s.io/kubernetes/pkg/scheduler"
@@ -75,6 +78,11 @@ func newPlugin(plugin framework.Plugin) frameworkruntime.PluginFactory {
75
78
}
76
79
}
77
80
81
+ type QueueSortPlugin struct {
82
+ // lessFunc is used to compare two queued pod infos.
83
+ lessFunc func (info1 , info2 * framework.QueuedPodInfo ) bool
84
+ }
85
+
78
86
type PreEnqueuePlugin struct {
79
87
called int
80
88
admit bool
@@ -282,6 +290,7 @@ func (pp *PermitPlugin) deepCopy() *PermitPlugin {
282
290
}
283
291
284
292
const (
293
+ queuesortPluginName = "queuesort-plugin"
285
294
enqueuePluginName = "enqueue-plugin"
286
295
prefilterPluginName = "prefilter-plugin"
287
296
postfilterPluginName = "postfilter-plugin"
@@ -308,6 +317,25 @@ var _ framework.PreBindPlugin = &PreBindPlugin{}
308
317
var _ framework.BindPlugin = & BindPlugin {}
309
318
var _ framework.PostBindPlugin = & PostBindPlugin {}
310
319
var _ framework.PermitPlugin = & PermitPlugin {}
320
+ var _ framework.QueueSortPlugin = & QueueSortPlugin {}
321
+
322
+ func (ep * QueueSortPlugin ) Name () string {
323
+ return queuesortPluginName
324
+ }
325
+
326
+ func (ep * QueueSortPlugin ) Less (info1 , info2 * framework.QueuedPodInfo ) bool {
327
+ if ep .lessFunc != nil {
328
+ return ep .lessFunc (info1 , info2 )
329
+ }
330
+ // If no custom less function is provided, default to return true.
331
+ return true
332
+ }
333
+
334
+ func NewQueueSortPlugin (lessFunc func (info1 , info2 * framework.QueuedPodInfo ) bool ) * QueueSortPlugin {
335
+ return & QueueSortPlugin {
336
+ lessFunc : lessFunc ,
337
+ }
338
+ }
311
339
312
340
func (ep * PreEnqueuePlugin ) Name () string {
313
341
return enqueuePluginName
@@ -668,7 +696,7 @@ func TestPreFilterPlugin(t *testing.T) {
668
696
preFilterPlugin := & PreFilterPlugin {}
669
697
registry , prof := initRegistryAndConfig (t , preFilterPlugin )
670
698
671
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
699
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
672
700
scheduler .WithProfiles (prof ),
673
701
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
674
702
defer teardown ()
@@ -705,6 +733,85 @@ func TestPreFilterPlugin(t *testing.T) {
705
733
}
706
734
}
707
735
736
+ // TestQueueSortPlugin tests invocation of queueSort plugins.
737
+ func TestQueueSortPlugin (t * testing.T ) {
738
+ tests := []struct {
739
+ name string
740
+ podNames []string
741
+ expectedOrder []string
742
+ customLessFunc func (info1 , info2 * framework.QueuedPodInfo ) bool
743
+ }{
744
+ {
745
+ name : "timestamp_sort_order" ,
746
+ podNames : []string {"pod-1" , "pod-2" , "pod-3" },
747
+ expectedOrder : []string {"pod-1" , "pod-2" , "pod-3" },
748
+ customLessFunc : func (info1 , info2 * framework.QueuedPodInfo ) bool {
749
+ return info1 .Timestamp .Before (info2 .Timestamp )
750
+ },
751
+ },
752
+ {
753
+ name : "priority_sort_order" ,
754
+ podNames : []string {"pod-1" , "pod-2" , "pod-3" },
755
+ expectedOrder : []string {"pod-3" , "pod-2" , "pod-1" }, // depends on pod priority
756
+ customLessFunc : func (info1 , info2 * framework.QueuedPodInfo ) bool {
757
+ p1 := corev1helpers .PodPriority (info1 .Pod )
758
+ p2 := corev1helpers .PodPriority (info2 .Pod )
759
+ return (p1 > p2 ) || (p1 == p2 && info1 .Timestamp .Before (info2 .Timestamp ))
760
+ },
761
+ },
762
+ }
763
+
764
+ for _ , tt := range tests {
765
+ t .Run (tt .name , func (t * testing.T ) {
766
+ queueSortPlugin := NewQueueSortPlugin (tt .customLessFunc )
767
+ registry , prof := initRegistryAndConfig (t , queueSortPlugin )
768
+
769
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "queuesort-plugin" , nil ), 2 , false ,
770
+ scheduler .WithProfiles (prof ),
771
+ scheduler .WithFrameworkOutOfTreeRegistry (registry ))
772
+ defer teardown ()
773
+
774
+ pods := make ([]* v1.Pod , 0 , len (tt .podNames ))
775
+ for i , name := range tt .podNames {
776
+ // Create a pod with different priority.
777
+ priority := int32 (i + 1 )
778
+ pod , err := testutils .CreatePausePod (testCtx .ClientSet ,
779
+ testutils .InitPausePod (& testutils.PausePodConfig {Name : name , Namespace : testCtx .NS .Name , Priority : & priority }))
780
+ if err != nil {
781
+ t .Fatalf ("Error while creating %v: %v" , name , err )
782
+ }
783
+ pods = append (pods , pod )
784
+ }
785
+
786
+ // Wait for all Pods to be in the scheduling queue.
787
+ err := wait .PollUntilContextTimeout (testCtx .Ctx , time .Millisecond * 200 , wait .ForeverTestTimeout , false , func (ctx context.Context ) (bool , error ) {
788
+ pendingPods , _ := testCtx .Scheduler .SchedulingQueue .PendingPods ()
789
+ if len (pendingPods ) == len (pods ) {
790
+ t .Logf ("All Pods are in the pending queue." )
791
+ return true , nil
792
+ }
793
+ t .Logf ("Waiting for all Pods to be in the scheduling queue. %d/%d" , len (pendingPods ), len (pods ))
794
+ return false , nil
795
+ })
796
+ if err != nil {
797
+ t .Fatalf ("Failed to observe all Pods in the scheduling queue: %v" , err )
798
+ }
799
+
800
+ actualOrder := make ([]string , len (tt .expectedOrder ))
801
+ for i := 0 ; i < len (tt .expectedOrder ); i ++ {
802
+ queueInfo := testutils .NextPodOrDie (t , testCtx )
803
+ actualOrder [i ] = queueInfo .Pod .Name
804
+ t .Logf ("Popped Pod %q" , queueInfo .Pod .Name )
805
+ }
806
+ if diff := cmp .Diff (tt .expectedOrder , actualOrder ); diff != "" {
807
+ t .Errorf ("Expected Pod order (-want,+got):\n %s" , diff )
808
+ } else {
809
+ t .Logf ("Pods were popped out in the expected order based on custom sorting logic." )
810
+ }
811
+ })
812
+ }
813
+ }
814
+
708
815
// TestPostFilterPlugin tests invocation of postFilter plugins.
709
816
func TestPostFilterPlugin (t * testing.T ) {
710
817
numNodes := 1
@@ -848,7 +955,7 @@ func TestPostFilterPlugin(t *testing.T) {
848
955
},
849
956
}}})
850
957
851
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , int (tt .numNodes ),
958
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , int (tt .numNodes ), true ,
852
959
scheduler .WithProfiles (cfg .Profiles ... ),
853
960
scheduler .WithFrameworkOutOfTreeRegistry (registry ),
854
961
)
@@ -916,7 +1023,7 @@ func TestScorePlugin(t *testing.T) {
916
1023
scorePlugin := & ScorePlugin {}
917
1024
registry , prof := initRegistryAndConfig (t , scorePlugin )
918
1025
919
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 10 ,
1026
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 10 , true ,
920
1027
scheduler .WithProfiles (prof ),
921
1028
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
922
1029
defer teardown ()
@@ -960,7 +1067,7 @@ func TestNormalizeScorePlugin(t *testing.T) {
960
1067
scoreWithNormalizePlugin := & ScoreWithNormalizePlugin {}
961
1068
registry , prof := initRegistryAndConfig (t , scoreWithNormalizePlugin )
962
1069
963
- testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "score-plugin" , nil ), 10 ,
1070
+ testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "score-plugin" , nil ), 10 , true ,
964
1071
scheduler .WithProfiles (prof ),
965
1072
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
966
1073
@@ -1008,7 +1115,7 @@ func TestReservePluginReserve(t *testing.T) {
1008
1115
reservePlugin := & ReservePlugin {}
1009
1116
registry , prof := initRegistryAndConfig (t , reservePlugin )
1010
1117
1011
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1118
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1012
1119
scheduler .WithProfiles (prof ),
1013
1120
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1014
1121
defer teardown ()
@@ -1123,7 +1230,7 @@ func TestPrebindPlugin(t *testing.T) {
1123
1230
},
1124
1231
})
1125
1232
1126
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , nodesNum ,
1233
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , nodesNum , true ,
1127
1234
scheduler .WithProfiles (cfg .Profiles ... ),
1128
1235
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1129
1236
defer teardown ()
@@ -1279,7 +1386,7 @@ func TestUnReserveReservePlugins(t *testing.T) {
1279
1386
}
1280
1387
registry , prof := initRegistryAndConfig (t , pls ... )
1281
1388
1282
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1389
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1283
1390
scheduler .WithProfiles (prof ),
1284
1391
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1285
1392
defer teardown ()
@@ -1373,7 +1480,7 @@ func TestUnReservePermitPlugins(t *testing.T) {
1373
1480
}
1374
1481
registry , profile := initRegistryAndConfig (t , []framework.Plugin {test .plugin , reservePlugin }... )
1375
1482
1376
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1483
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1377
1484
scheduler .WithProfiles (profile ),
1378
1485
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1379
1486
defer teardown ()
@@ -1445,7 +1552,7 @@ func TestUnReservePreBindPlugins(t *testing.T) {
1445
1552
}
1446
1553
registry , profile := initRegistryAndConfig (t , []framework.Plugin {test .plugin , reservePlugin }... )
1447
1554
1448
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1555
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1449
1556
scheduler .WithProfiles (profile ),
1450
1557
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1451
1558
defer teardown ()
@@ -1516,7 +1623,7 @@ func TestUnReserveBindPlugins(t *testing.T) {
1516
1623
1517
1624
test .plugin .client = testContext .ClientSet
1518
1625
1519
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1626
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1520
1627
scheduler .WithProfiles (profile ),
1521
1628
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1522
1629
defer teardown ()
@@ -1663,7 +1770,7 @@ func TestBindPlugin(t *testing.T) {
1663
1770
}},
1664
1771
})
1665
1772
1666
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1773
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1667
1774
scheduler .WithProfiles (cfg .Profiles ... ),
1668
1775
scheduler .WithFrameworkOutOfTreeRegistry (registry ),
1669
1776
)
@@ -1789,7 +1896,7 @@ func TestPostBindPlugin(t *testing.T) {
1789
1896
}
1790
1897
1791
1898
registry , prof := initRegistryAndConfig (t , preBindPlugin , postBindPlugin )
1792
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1899
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1793
1900
scheduler .WithProfiles (prof ),
1794
1901
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1795
1902
defer teardown ()
@@ -1881,7 +1988,7 @@ func TestPermitPlugin(t *testing.T) {
1881
1988
perPlugin := & PermitPlugin {name : permitPluginName }
1882
1989
registry , prof := initRegistryAndConfig (t , perPlugin )
1883
1990
1884
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
1991
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
1885
1992
scheduler .WithProfiles (prof ),
1886
1993
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1887
1994
defer teardown ()
@@ -1931,7 +2038,7 @@ func TestMultiplePermitPlugins(t *testing.T) {
1931
2038
registry , prof := initRegistryAndConfig (t , perPlugin1 , perPlugin2 )
1932
2039
1933
2040
// Create the API server and the scheduler with the test plugin set.
1934
- testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "multi-permit-plugin" , nil ), 2 ,
2041
+ testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "multi-permit-plugin" , nil ), 2 , true ,
1935
2042
scheduler .WithProfiles (prof ),
1936
2043
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1937
2044
@@ -1983,7 +2090,7 @@ func TestPermitPluginsCancelled(t *testing.T) {
1983
2090
registry , prof := initRegistryAndConfig (t , perPlugin1 , perPlugin2 )
1984
2091
1985
2092
// Create the API server and the scheduler with the test plugin set.
1986
- testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "permit-plugins" , nil ), 2 ,
2093
+ testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "permit-plugins" , nil ), 2 , true ,
1987
2094
scheduler .WithProfiles (prof ),
1988
2095
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
1989
2096
@@ -2046,7 +2153,7 @@ func TestCoSchedulingWithPermitPlugin(t *testing.T) {
2046
2153
permitPlugin := & PermitPlugin {name : permitPluginName }
2047
2154
registry , prof := initRegistryAndConfig (t , permitPlugin )
2048
2155
2049
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
2156
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
2050
2157
scheduler .WithProfiles (prof ),
2051
2158
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
2052
2159
defer teardown ()
@@ -2128,7 +2235,7 @@ func TestFilterPlugin(t *testing.T) {
2128
2235
filterPlugin := & FilterPlugin {}
2129
2236
registry , prof := initRegistryAndConfig (t , filterPlugin )
2130
2237
2131
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 1 ,
2238
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 1 , true ,
2132
2239
scheduler .WithProfiles (prof ),
2133
2240
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
2134
2241
defer teardown ()
@@ -2185,7 +2292,7 @@ func TestPreScorePlugin(t *testing.T) {
2185
2292
preScorePlugin := & PreScorePlugin {}
2186
2293
registry , prof := initRegistryAndConfig (t , preScorePlugin )
2187
2294
2188
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
2295
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
2189
2296
scheduler .WithProfiles (prof ),
2190
2297
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
2191
2298
defer teardown ()
@@ -2245,7 +2352,7 @@ func TestPreEnqueuePlugin(t *testing.T) {
2245
2352
preFilterPlugin := & PreFilterPlugin {}
2246
2353
registry , prof := initRegistryAndConfig (t , enqueuePlugin , preFilterPlugin )
2247
2354
2248
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 1 ,
2355
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 1 , true ,
2249
2356
scheduler .WithProfiles (prof ),
2250
2357
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
2251
2358
defer teardown ()
@@ -2377,7 +2484,7 @@ func TestPreemptWithPermitPlugin(t *testing.T) {
2377
2484
},
2378
2485
})
2379
2486
2380
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 0 ,
2487
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 0 , true ,
2381
2488
scheduler .WithProfiles (cfg .Profiles ... ),
2382
2489
scheduler .WithFrameworkOutOfTreeRegistry (registry ),
2383
2490
)
@@ -2557,7 +2664,7 @@ func TestActivatePods(t *testing.T) {
2557
2664
})
2558
2665
2559
2666
// Create the API server and the scheduler with the test plugin set.
2560
- testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "job-plugin" , nil ), 1 ,
2667
+ testCtx , _ := schedulerutils .InitTestSchedulerForFrameworkTest (t , testutils .InitTestAPIServer (t , "job-plugin" , nil ), 1 , true ,
2561
2668
scheduler .WithProfiles (cfg .Profiles ... ),
2562
2669
scheduler .WithFrameworkOutOfTreeRegistry (registry ))
2563
2670
@@ -2730,7 +2837,7 @@ func TestPreEnqueuePluginEventsToRegister(t *testing.T) {
2730
2837
}},
2731
2838
})
2732
2839
2733
- testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 ,
2840
+ testCtx , teardown := schedulerutils .InitTestSchedulerForFrameworkTest (t , testContext , 2 , true ,
2734
2841
scheduler .WithProfiles (cfg .Profiles ... ),
2735
2842
scheduler .WithFrameworkOutOfTreeRegistry (registry ),
2736
2843
)
0 commit comments