Merge pull request kubernetes#83460 from ahg-g/ahg-first-predicate

[migration phase 1] Implement PodToleratesNodeTaint as a filter plugin

Author: k8s-ci-robot

pkg/scheduler/algorithm/predicates/error.go

@@ -82,6 +82,38 @@ var (
 	ErrFakePredicate = newPredicateFailureError("FakePredicateError", "Nodes failed the fake predicate")
 )
 
+var unresolvablePredicateFailureErrors = map[PredicateFailureReason]struct{}{
+	ErrNodeSelectorNotMatch:      {},
+	ErrPodAffinityRulesNotMatch:  {},
+	ErrPodNotMatchHostName:       {},
+	ErrTaintsTolerationsNotMatch: {},
+	ErrNodeLabelPresenceViolated: {},
+	// Node conditions won't change when scheduler simulates removal of preemption victims.
+	// So, it is pointless to try nodes that have not been able to host the pod due to node
+	// conditions. These include ErrNodeNotReady, ErrNodeUnderPIDPressure, ErrNodeUnderMemoryPressure, ....
+	ErrNodeNotReady:            {},
+	ErrNodeNetworkUnavailable:  {},
+	ErrNodeUnderDiskPressure:   {},
+	ErrNodeUnderPIDPressure:    {},
+	ErrNodeUnderMemoryPressure: {},
+	ErrNodeUnschedulable:       {},
+	ErrNodeUnknownCondition:    {},
+	ErrVolumeZoneConflict:      {},
+	ErrVolumeNodeConflict:      {},
+	ErrVolumeBindConflict:      {},
+}
+
+// UnresolvablePredicateExists checks if there is at least one unresolvable predicate failure reason, if true
+// returns the first one in the list.
+func UnresolvablePredicateExists(reasons []PredicateFailureReason) PredicateFailureReason {
+	for _, r := range reasons {
+		if _, ok := unresolvablePredicateFailureErrors[r]; ok {
+			return r
+		}
+	}
+	return nil
+}
+
 // InsufficientResourceError is an error type that indicates what kind of resource limit is
 // hit and caused the unfitting failure.
 type InsufficientResourceError struct {

pkg/scheduler/api/compatibility/compatibility_test.go

@@ -37,10 +37,11 @@ import (
 func TestCompatibility_v1_Scheduler(t *testing.T) {
 	// Add serialized versions of scheduler config that exercise available options to ensure compatibility between releases
 	schedulerFiles := map[string]struct {
-		JSON             string
-		wantPredicates   sets.String
-		wantPrioritizers sets.String
-		wantExtenders    []schedulerapi.ExtenderConfig
+		JSON              string
+		wantPredicates    sets.String
+		wantPrioritizers  sets.String
+		wantFilterPlugins sets.String
+		wantExtenders     []schedulerapi.ExtenderConfig
 	}{
 		// Do not change this JSON after the corresponding release has been tagged.
 		// A failure indicates backwards compatibility with the specified release was broken.
@@ -214,7 +215,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"MaxEBSVolumeCount",
 				"MaxGCEPDVolumeCount",
@@ -234,6 +234,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"TaintTolerationPriority",
 				"InterPodAffinityPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 		},
 
 		// Do not change this JSON after the corresponding release has been tagged.
@@ -279,7 +282,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"MaxEBSVolumeCount",
@@ -302,6 +304,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 		},
 		// Do not change this JSON after the corresponding release has been tagged.
 		// A failure indicates backwards compatibility with the specified release was broken.
@@ -356,7 +361,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"MaxEBSVolumeCount",
@@ -379,6 +383,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -445,7 +452,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"CheckNodeCondition",
@@ -469,6 +475,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -536,7 +545,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"CheckNodeCondition",
@@ -561,6 +569,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -632,7 +643,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"CheckNodePIDPressure",
@@ -658,6 +668,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"InterPodAffinityPriority",
 				"MostRequestedPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -741,7 +754,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"CheckNodePIDPressure",
@@ -768,6 +780,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -852,7 +867,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"CheckNodePIDPressure",
@@ -880,6 +894,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -963,7 +980,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"CheckNodePIDPressure",
@@ -992,6 +1008,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -1079,7 +1098,6 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"HostName",
 				"NoDiskConflict",
 				"NoVolumeZoneConflict",
-				"PodToleratesNodeTaints",
 				"CheckNodeMemoryPressure",
 				"CheckNodeDiskPressure",
 				"CheckNodePIDPressure",
@@ -1108,6 +1126,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				"MostRequestedPriority",
 				"RequestedToCapacityRatioPriority",
 			),
+			wantFilterPlugins: sets.NewString(
+				"TaintToleration",
+			),
 			wantExtenders: []schedulerapi.ExtenderConfig{{
 				URLPrefix:        "/prefix",
 				FilterVerb:       "filter",
@@ -1128,6 +1149,9 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 	seenPredicates := sets.NewString()
 	seenPriorities := sets.NewString()
 	mandatoryPredicates := sets.NewString("CheckNodeCondition")
+	filterToPredicateMap := map[string]string{
+		"TaintToleration": "PodToleratesNodeTaints",
+	}
 
 	for v, tc := range schedulerFiles {
 		t.Run(v, func(t *testing.T) {
@@ -1163,26 +1187,39 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				algorithmSrc,
 				make(chan struct{}),
 			)
+
 			if err != nil {
 				t.Fatalf("%s: Error constructing: %v", v, err)
 			}
-			schedPredicates := sets.NewString()
+			gotPredicates := sets.NewString()
 			for p := range sched.Algorithm.Predicates() {
-				schedPredicates.Insert(p)
+				gotPredicates.Insert(p)
 			}
 			wantPredicates := tc.wantPredicates.Union(mandatoryPredicates)
-			if !schedPredicates.Equal(wantPredicates) {
-				t.Errorf("Got predicates %v, want %v", schedPredicates, wantPredicates)
+			if !gotPredicates.Equal(wantPredicates) {
+				t.Errorf("Got predicates %v, want %v", gotPredicates, wantPredicates)
 			}
-			schedPrioritizers := sets.NewString()
+
+			gotPrioritizers := sets.NewString()
 			for _, p := range sched.Algorithm.Prioritizers() {
-				schedPrioritizers.Insert(p.Name)
+				gotPrioritizers.Insert(p.Name)
+			}
+			if !gotPrioritizers.Equal(tc.wantPrioritizers) {
+				t.Errorf("Got prioritizers %v, want %v", gotPrioritizers, tc.wantPrioritizers)
 			}
 
-			if !schedPrioritizers.Equal(tc.wantPrioritizers) {
-				t.Errorf("Got prioritizers %v, want %v", schedPrioritizers, tc.wantPrioritizers)
+			gotFilterPlugins := sets.NewString()
+			plugins := sched.Framework.ListPlugins()
+			for _, p := range plugins["FilterPlugin"] {
+				gotFilterPlugins.Insert(p)
+				seenPredicates.Insert(filterToPredicateMap[p])
+
+			}
+			if !gotFilterPlugins.Equal(tc.wantFilterPlugins) {
+				t.Errorf("Got filter plugins %v, want %v", gotFilterPlugins, tc.wantFilterPlugins)
 			}
-			schedExtenders := sched.Algorithm.Extenders()
+
+			gotExtenders := sched.Algorithm.Extenders()
 			var wantExtenders []*core.HTTPExtender
 			for _, e := range tc.wantExtenders {
 				extender, err := core.NewHTTPExtender(&e)
@@ -1191,13 +1228,14 @@ func TestCompatibility_v1_Scheduler(t *testing.T) {
 				}
 				wantExtenders = append(wantExtenders, extender.(*core.HTTPExtender))
 			}
-			for i := range schedExtenders {
-				if !core.Equal(wantExtenders[i], schedExtenders[i].(*core.HTTPExtender)) {
-					t.Errorf("Got extender #%d %+v, want %+v", i, schedExtenders[i], wantExtenders[i])
+			for i := range gotExtenders {
+				if !core.Equal(wantExtenders[i], gotExtenders[i].(*core.HTTPExtender)) {
+					t.Errorf("Got extender #%d %+v, want %+v", i, gotExtenders[i], wantExtenders[i])
 				}
 			}
-			seenPredicates = seenPredicates.Union(schedPredicates)
-			seenPriorities = seenPriorities.Union(schedPrioritizers)
+
+			seenPredicates = seenPredicates.Union(gotPredicates)
+			seenPriorities = seenPriorities.Union(gotPrioritizers)
 		})
 	}
 

pkg/scheduler/core/generic_scheduler.go

@@ -65,27 +65,6 @@ const (
 	minFeasibleNodesPercentageToFind = 5
 )
 
-var unresolvablePredicateFailureErrors = map[predicates.PredicateFailureReason]struct{}{
-	predicates.ErrNodeSelectorNotMatch:      {},
-	predicates.ErrPodAffinityRulesNotMatch:  {},
-	predicates.ErrPodNotMatchHostName:       {},
-	predicates.ErrTaintsTolerationsNotMatch: {},
-	predicates.ErrNodeLabelPresenceViolated: {},
-	// Node conditions won't change when scheduler simulates removal of preemption victims.
-	// So, it is pointless to try nodes that have not been able to host the pod due to node
-	// conditions. These include ErrNodeNotReady, ErrNodeUnderPIDPressure, ErrNodeUnderMemoryPressure, ....
-	predicates.ErrNodeNotReady:            {},
-	predicates.ErrNodeNetworkUnavailable:  {},
-	predicates.ErrNodeUnderDiskPressure:   {},
-	predicates.ErrNodeUnderPIDPressure:    {},
-	predicates.ErrNodeUnderMemoryPressure: {},
-	predicates.ErrNodeUnschedulable:       {},
-	predicates.ErrNodeUnknownCondition:    {},
-	predicates.ErrVolumeZoneConflict:      {},
-	predicates.ErrVolumeNodeConflict:      {},
-	predicates.ErrVolumeBindConflict:      {},
-}
-
 // FailedPredicateMap declares a map[string][]algorithm.PredicateFailureReason type.
 type FailedPredicateMap map[string][]predicates.PredicateFailureReason
 
@@ -1204,16 +1183,6 @@ func (g *genericScheduler) selectVictimsOnNode(
 	return victims, numViolatingVictim, true
 }
 
-// unresolvablePredicateExists checks whether failedPredicates has unresolvable predicate.
-func unresolvablePredicateExists(failedPredicates []predicates.PredicateFailureReason) bool {
-	for _, failedPredicate := range failedPredicates {
-		if _, ok := unresolvablePredicateFailureErrors[failedPredicate]; ok {
-			return true
-		}
-	}
-	return false
-}
-
 // nodesWherePreemptionMightHelp returns a list of nodes with failed predicates
 // that may be satisfied by removing pods from the node.
 func nodesWherePreemptionMightHelp(nodes []*v1.Node, fitErr *FitError) []*v1.Node {
@@ -1228,7 +1197,7 @@ func nodesWherePreemptionMightHelp(nodes []*v1.Node, fitErr *FitError) []*v1.Nod
 		// to rely less on such assumptions in the code when checking does not impose
 		// significant overhead.
 		// Also, we currently assume all failures returned by extender as resolvable.
-		if !unresolvablePredicateExists(failedPredicates) {
+		if predicates.UnresolvablePredicateExists(failedPredicates) == nil {
 			klog.V(3).Infof("Node %v is a potential node for preemption.", node.Name)
 			potentialNodes = append(potentialNodes, node)
 		}

pkg/scheduler/framework/plugins/BUILD

@@ -6,8 +6,9 @@ go_library(
     importpath = "k8s.io/kubernetes/pkg/scheduler/framework/plugins",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/scheduler/algorithm/predicates:go_default_library",
         "//pkg/scheduler/apis/config:go_default_library",
-        "//pkg/scheduler/framework/plugins/noop:go_default_library",
+        "//pkg/scheduler/framework/plugins/tainttoleration:go_default_library",
         "//pkg/scheduler/framework/v1alpha1:go_default_library",
     ],
 )
@@ -24,7 +25,9 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//pkg/scheduler/framework/plugins/examples:all-srcs",
+        "//pkg/scheduler/framework/plugins/migration:all-srcs",
         "//pkg/scheduler/framework/plugins/noop:all-srcs",
+        "//pkg/scheduler/framework/plugins/tainttoleration:all-srcs",
     ],
     tags = ["automanaged"],
     visibility = ["//visibility:public"],