Merge pull request kubernetes#77501 from JieJhih/scheduling/plugin

add scheduling framework configuration

Author: k8s-ci-robot

cmd/kube-scheduler/app/options/BUILD

@@ -70,6 +70,7 @@ go_test(
         "//cmd/kube-scheduler/app/config:go_default_library",
         "//pkg/scheduler/apis/config:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/rand:go_default_library",
         "//staging/src/k8s.io/apiserver/pkg/server/options:go_default_library",

cmd/kube-scheduler/app/options/options_test.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/diff"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
 	componentbaseconfig "k8s.io/component-base/config"
@@ -146,6 +147,31 @@ users:
 		t.Fatal(err)
 	}
 
+	// plugin config
+	pluginconfigFile := filepath.Join(tmpDir, "plugin.yaml")
+	if err := ioutil.WriteFile(pluginconfigFile, []byte(fmt.Sprintf(`
+apiVersion: kubescheduler.config.k8s.io/v1alpha1
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: "%s"
+plugins:
+  reserve:
+    enabled:
+    - name: foo
+    - name: bar
+    disabled:
+    - name: baz
+  preBind:
+    enabled:
+    - name: foo
+    disabled:
+    - name: baz
+pluginConfig:
+- name: foo
+`, configKubeconfig)), os.FileMode(0600)); err != nil {
+		t.Fatal(err)
+	}
+
 	// Insulate this test from picking up in-cluster config when run inside a pod
 	// We can't assume we have permissions to write to /var/run/secrets/... from a unit test to mock in-cluster config for testing
 	originalHost := os.Getenv("KUBERNETES_SERVICE_HOST")
@@ -224,6 +250,7 @@ users:
 					ContentType: "application/vnd.kubernetes.protobuf",
 				},
 				BindTimeoutSeconds: &defaultBindTimeoutSeconds,
+				Plugins:            nil,
 			},
 		},
 		{
@@ -334,6 +361,73 @@ users:
 			},
 			expectedUsername: "none, http",
 		},
+		{
+			name: "plugin config",
+			options: &Options{
+				ConfigFile: pluginconfigFile,
+			},
+			expectedUsername: "config",
+			expectedConfig: kubeschedulerconfig.KubeSchedulerConfiguration{
+				SchedulerName:                  "default-scheduler",
+				AlgorithmSource:                kubeschedulerconfig.SchedulerAlgorithmSource{Provider: &defaultSource},
+				HardPodAffinitySymmetricWeight: 1,
+				HealthzBindAddress:             "0.0.0.0:10251",
+				MetricsBindAddress:             "0.0.0.0:10251",
+				LeaderElection: kubeschedulerconfig.KubeSchedulerLeaderElectionConfiguration{
+					LeaderElectionConfiguration: componentbaseconfig.LeaderElectionConfiguration{
+						LeaderElect:   true,
+						LeaseDuration: metav1.Duration{Duration: 15 * time.Second},
+						RenewDeadline: metav1.Duration{Duration: 10 * time.Second},
+						RetryPeriod:   metav1.Duration{Duration: 2 * time.Second},
+						ResourceLock:  "endpoints",
+					},
+					LockObjectNamespace: "kube-system",
+					LockObjectName:      "kube-scheduler",
+				},
+				ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
+					Kubeconfig:  configKubeconfig,
+					QPS:         50,
+					Burst:       100,
+					ContentType: "application/vnd.kubernetes.protobuf",
+				},
+				BindTimeoutSeconds: &defaultBindTimeoutSeconds,
+				Plugins: &kubeschedulerconfig.Plugins{
+					Reserve: &kubeschedulerconfig.PluginSet{
+						Enabled: []kubeschedulerconfig.Plugin{
+							{
+								Name: "foo",
+							},
+							{
+								Name: "bar",
+							},
+						},
+						Disabled: []kubeschedulerconfig.Plugin{
+							{
+								Name: "baz",
+							},
+						},
+					},
+					PreBind: &kubeschedulerconfig.PluginSet{
+						Enabled: []kubeschedulerconfig.Plugin{
+							{
+								Name: "foo",
+							},
+						},
+						Disabled: []kubeschedulerconfig.Plugin{
+							{
+								Name: "baz",
+							},
+						},
+					},
+				},
+				PluginConfig: []kubeschedulerconfig.PluginConfig{
+					{
+						Name: "foo",
+						Args: runtime.Unknown{},
+					},
+				},
+			},
+		},
 		{
 			name:          "no config",
 			options:       &Options{},

cmd/kube-scheduler/app/server.go

@@ -176,6 +176,8 @@ func Run(cc schedulerserverconfig.CompletedConfig, stopCh <-chan struct{}) error
 		cc.ComponentConfig.AlgorithmSource,
 		stopCh,
 		framework.NewRegistry(),
+		cc.ComponentConfig.Plugins,
+		cc.ComponentConfig.PluginConfig,
 		scheduler.WithName(cc.ComponentConfig.SchedulerName),
 		scheduler.WithHardPodAffinitySymmetricWeight(cc.ComponentConfig.HardPodAffinitySymmetricWeight),
 		scheduler.WithPreemptionDisabled(cc.ComponentConfig.DisablePreemption),

pkg/scheduler/apis/config/types.go

@@ -18,6 +18,7 @@ package config
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	componentbaseconfig "k8s.io/component-base/config"
 )
 
@@ -86,6 +87,17 @@ type KubeSchedulerConfiguration struct {
 	// Value must be non-negative integer. The value zero indicates no waiting.
 	// If this value is nil, the default value will be used.
 	BindTimeoutSeconds *int64
+
+	// Plugins specify the set of plugins that should be enabled or disabled. Enabled plugins are the
+	// ones that should be enabled in addition to the default plugins. Disabled plugins are any of the
+	// default plugins that should be disabled.
+	// When no enabled or disabled plugin is specified for an extension point, default plugins for
+	// that extension point will be used if there is any.
+	Plugins *Plugins
+
+	// PluginConfig is an optional set of custom plugin arguments for each plugin.
+	// Omitting config args for a plugin is equivalent to using the default config for that plugin.
+	PluginConfig []PluginConfig
 }
 
 // SchedulerAlgorithmSource is the source of a scheduler algorithm. One source
@@ -131,3 +143,76 @@ type KubeSchedulerLeaderElectionConfiguration struct {
 	// LockObjectName defines the lock object name
 	LockObjectName string
 }
+
+// Plugins include multiple extension points. When specified, the list of plugins for
+// a particular extension point are the only ones enabled. If an extension point is
+// omitted from the config, then the default set of plugins is used for that extension point.
+// Enabled plugins are called in the order specified here, after default plugins. If they need to
+// be invoked before default plugins, default plugins must be disabled and re-enabled here in desired order.
+type Plugins struct {
+	// QueueSort is a list of plugins that should be invoked when sorting pods in the scheduling queue.
+	QueueSort *PluginSet
+
+	// PreFilter is a list of plugins that should be invoked at "PreFilter" extension point of the scheduling framework.
+	PreFilter *PluginSet
+
+	// Filter is a list of plugins that should be invoked when filtering out nodes that cannot run the Pod.
+	Filter *PluginSet
+
+	// PostFilter is a list of plugins that are invoked after filtering out infeasible nodes.
+	PostFilter *PluginSet
+
+	// Score is a list of plugins that should be invoked when ranking nodes that have passed the filtering phase.
+	Score *PluginSet
+
+	// NormalizeScore is a list of plugins that should be invoked after the scoring phase to normalize scores.
+	NormalizeScore *PluginSet
+
+	// Reserve is a list of plugins invoked when reserving a node to run the pod.
+	Reserve *PluginSet
+
+	// Permit is a list of plugins that control binding of a Pod. These plugins can prevent or delay binding of a Pod.
+	Permit *PluginSet
+
+	// PreBind is a list of plugins that should be invoked before a pod is bound.
+	PreBind *PluginSet
+
+	// Bind is a list of plugins that should be invoked at "Bind" extension point of the scheduling framework.
+	// The scheduler call these plugins in order. Scheduler skips the rest of these plugins as soon as one returns success.
+	Bind *PluginSet
+
+	// PostBind is a list of plugins that should be invoked after a pod is successfully bound.
+	PostBind *PluginSet
+
+	// Unreserve is a list of plugins invoked when a pod that was previously reserved is rejected in a later phase.
+	Unreserve *PluginSet
+}
+
+// PluginSet specifies enabled and disabled plugins for an extension point.
+// If an array is empty, missing, or nil, default plugins at that extension point will be used.
+type PluginSet struct {
+	// Enabled specifies plugins that should be enabled in addition to default plugins.
+	// These are called after default plugins and in the same order specified here.
+	Enabled []Plugin
+	// Disabled specifies default plugins that should be disabled.
+	// When all default plugins need to be disabled, an array containing only one "*" should be provided.
+	Disabled []Plugin
+}
+
+// Plugin specifies a plugin name and its weight when applicable. Weight is used only for Score plugins.
+type Plugin struct {
+	// Name defines the name of plugin
+	Name string
+	// Weight defines the weight of plugin, only used for Score plugins.
+	Weight int32
+}
+
+// PluginConfig specifies arguments that should be passed to a plugin at the time of initialization.
+// A plugin that is invoked at multiple extension points is initialized once. Args can have arbitrary structure.
+// It is up to the plugin to process these Args.
+type PluginConfig struct {
+	// Name defines the name of plugin being configured
+	Name string
+	// Args defines the arguments passed to the plugins at the time of initialization. Args can have arbitrary structure.
+	Args runtime.Unknown
+}