perf(system): migrate optimizations for high-water-level and unschedulable scenarios

cmd/scheduler/app/options/options_test.go

@@ -38,10 +38,11 @@ var (
 	FLOAT64_3  = 3.0
 	FLOAT64_10 = 10.0
 
-	INT32_0  = int32(0)
-	INT32_10 = int32(10)
-	INT32_20 = int32(20)
-	INT32_40 = int32(40)
+	INT32_0    = int32(0)
+	INT32_10   = int32(10)
+	INT32_20   = int32(20)
+	INT32_40   = int32(40)
+	INT32_1000 = int32(1000)
 
 	INT64_1   = int64(1)
 	INT64_2   = int64(2)
@@ -204,6 +205,7 @@ func TestLoadFileV1beta1(t *testing.T) {
 					},
 				},
 			},
+			ExpectedThroughput:                &INT32_1000,
 			PercentageOfNodesToScore:          &INT32_0,
 			IncreasedPercentageOfNodesToScore: &INT32_0,
 			DisablePreemption:                 &FALSE,

pkg/features/godel_features.go

@@ -84,6 +84,12 @@ const (
 	//
 	// Allows to trigger resource reservation in Godel.
 	ResourceReservation featuregate.Feature = "ResourceReservation"
+
+	// owner: @libing.binacs
+	// alpha: for now
+	//
+	// support skipping filtering unchanged nodes.
+	SkipFilteringUnchangedNodes featuregate.Feature = "SkipFilteringUnchangedNodes"
 )
 
 func init() {
@@ -104,4 +110,5 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	EnableColocation:                        {Default: false, PreRelease: featuregate.Alpha},
 	SupportRescheduling:                     {Default: false, PreRelease: featuregate.Alpha},
 	ResourceReservation:                     {Default: false, PreRelease: featuregate.Alpha},
+	SkipFilteringUnchangedNodes:             {Default: false, PreRelease: featuregate.Alpha},
 }

pkg/framework/api/common_state.go

@@ -30,6 +30,7 @@ const (
 	NodePartitionTypeStateKey = "NodePartitionType"
 	PodLauncherStateKey       = "PodLauncher"
 	PodResourceTypeStateKey   = "PodResourceType"
+	PodSchedulingCtxKey       = "PodSchedulingCtx"
 	PodTraceStateKey          = "PodTrace"
 	NodeGroupStateKey         = "NodeGroup"
 	PotentialVictimsKey       = "PotentialVictims"
@@ -45,6 +46,7 @@ const (
 	NodePartitionTypeMissedErrorString = "failed to get NodePartitionType, supposed to be set in cycle state"
 	PodLauncherMissedErrorString       = "failed to get PodLauncher, supposed to be set in cycle state"
 	PodResourceTypeMissingErrorString  = "failed to get PodResourceType, supposed to be set in cycle state"
+	PodSchedulingCtxMissingErrorString = "failed to get PodSchedulingCtx, supposed to be set in cycle state"
 	PodTraceMissingErrorString         = "failed to get PodTrace, supposed to be set in cycle state"
 	NodeGroupMissedErrorString         = "failed to get NodeGroup, supposed to be set in cycle state"
 
@@ -91,6 +93,17 @@ func SetPodLauncherState(podLauncher podutil.PodLauncher, state *CycleState) err
 	return podutil.PodLauncherUnsupportError
 }
 
+func SetPodSchedulingCtxKey(schedulingCtx *PodSchedulingCtx, state *CycleState) error {
+	if schedulingCtx == nil {
+		schedulingCtx = &PodSchedulingCtx{}
+	}
+	data := &stateData{
+		data: schedulingCtx,
+	}
+	state.Write(PodSchedulingCtxKey, data)
+	return nil
+}
+
 func SetPodTrace(podTrace tracing.SchedulingTrace, state *CycleState) error {
 	data := &stateData{
 		data: podTrace,
@@ -242,6 +255,17 @@ func GetPodLauncher(state *CycleState) (podutil.PodLauncher, error) {
 	return "", PodLauncherMissedError
 }
 
+var PodSchedulingCtxMissingError = fmt.Errorf(PodSchedulingCtxMissingErrorString)
+
+func GetPodSchedulingCtx(state *CycleState) (*PodSchedulingCtx, error) {
+	if data, err := state.Read(PodSchedulingCtxKey); err == nil {
+		if s, ok := data.(*stateData); ok {
+			return s.data.(*PodSchedulingCtx), nil
+		}
+	}
+	return nil, PodSchedulingCtxMissingError
+}
+
 var PodTraceMissingError = fmt.Errorf(PodTraceMissingErrorString)
 
 func GetPodTrace(state *CycleState) (tracing.SchedulingTrace, error) {

pkg/framework/api/types.go

@@ -99,6 +99,10 @@ type ClusterEvent struct {
 // RecorderFactory builds an EventRecorder for a given scheduler name.
 type RecorderFactory func(string) events.EventRecorder
 
+type PodSchedulingCtx struct {
+	NodeStoreGeneration int64
+}
+
 // QueuedPodInfo is a Pod wrapper with additional information related to
 // the pod's status in the scheduling queue, such as the timestamp when
 // it's added to the queue.
@@ -121,6 +125,8 @@ type QueuedPodInfo struct {
 	// takes more time.
 	InitialPreemptAttemptTimestamp time.Time
 
+	SchedulingCtx *PodSchedulingCtx
+
 	// Fields below are only used by binder, need to move this out of QueuedPodInfo and put them into BinderQueuedPodInfo
 	// TODO: (liumeng) implement BinderQueuedPodInfo ?
 	// assumedPod

pkg/scheduler/apis/config/defaults.go

@@ -150,6 +150,10 @@ func SetDefaults_GodelSchedulerConfiguration(obj *GodelSchedulerConfiguration) {
 			// We got SubClusterName "" as default.
 			obj.DefaultProfile = &GodelSchedulerProfile{}
 		}
+		if obj.DefaultProfile.ExpectedThroughput == nil {
+			expectedThroughput := int32(DefaultExpectedThroughput)
+			obj.DefaultProfile.ExpectedThroughput = &expectedThroughput
+		}
 		if obj.DefaultProfile.PercentageOfNodesToScore == nil {
 			percentageOfNodesToScore := int32(DefaultPercentageOfNodesToScore)
 			obj.DefaultProfile.PercentageOfNodesToScore = &percentageOfNodesToScore

pkg/scheduler/apis/config/types.go

@@ -62,6 +62,8 @@ const (
 )
 
 const (
+	DefaultExpectedThroughput = 0
+
 	// DefaultPercentageOfNodesToScore defines the percentage of nodes of all nodes
 	// that once found feasible, the scheduler stops looking for more nodes.
 	// A value of 0 means adaptive, meaning the scheduler figures out a proper default.
@@ -203,6 +205,8 @@ type GodelSchedulerProfile struct {
 	// for that preemption plugin.
 	PreemptionPluginConfigs []PluginConfig
 
+	ExpectedThroughput *int32
+
 	// TODO: reserve temporarily(godel).
 	// PercentageOfNodesToScore is the percentage of all nodes that once found feasible
 	// for running a pod, the scheduler stops its search for more feasible nodes in

pkg/scheduler/apis/config/v1beta1/defaults.go

@@ -150,6 +150,10 @@ func SetDefaults_GodelSchedulerConfiguration(obj *GodelSchedulerConfiguration) {
 			// We got SubClusterName "" as default.
 			obj.DefaultProfile = &GodelSchedulerProfile{}
 		}
+		if obj.DefaultProfile.ExpectedThroughput == nil {
+			expectedThroughput := int32(config.DefaultExpectedThroughput)
+			obj.DefaultProfile.ExpectedThroughput = &expectedThroughput
+		}
 		if obj.DefaultProfile.PercentageOfNodesToScore == nil {
 			percentageOfNodesToScore := int32(config.DefaultPercentageOfNodesToScore)
 			obj.DefaultProfile.PercentageOfNodesToScore = &percentageOfNodesToScore

pkg/scheduler/apis/config/v1beta1/types.go

@@ -161,6 +161,8 @@ type GodelSchedulerProfile struct {
 	// for that preemption plugin.
 	PreemptionPluginConfigs []config.PluginConfig `json:"preemptionPluginConfigs,omitempty"`
 
+	ExpectedThroughput *int32 `json:"expectedThroughput,omitempty"`
+
 	// TODO: reserve temporarily(godel).
 	// PercentageOfNodesToScore is the percentage of all nodes that once found feasible
 	// for running a pod, the scheduler stops its search for more feasible nodes in

pkg/scheduler/apis/config/validation/validation.go

@@ -141,6 +141,10 @@ func ValidateSubClusterArgs(cc *config.GodelSchedulerProfile, fldPath *field.Pat
 	errs = append(errs, ValidateBasePluginsConfiguration(cc.BasePluginsForNM, field.NewPath("baseNMPlugins"))...)
 	errs = append(errs, ValidatePluginArgsConfiguration(cc.PluginConfigs, field.NewPath("pluginConfig"))...)
 
+	if cc.ExpectedThroughput != nil && *cc.ExpectedThroughput < 0 {
+		errs = append(errs, field.Invalid(field.NewPath("expectedThroughput"),
+			cc.ExpectedThroughput, "not in valid range [0-INF]"))
+	}
 	if cc.PercentageOfNodesToScore != nil && (*cc.PercentageOfNodesToScore < 0 || *cc.PercentageOfNodesToScore > 100) {
 		errs = append(errs, field.Invalid(field.NewPath("percentageOfNodesToScore"),
 			cc.PercentageOfNodesToScore, "not in valid range [0-100]"))

pkg/scheduler/cache/snapshot.go

@@ -24,6 +24,7 @@ import (
 	framework "github.com/kubewharf/godel-scheduler/pkg/framework/api"
 	"github.com/kubewharf/godel-scheduler/pkg/scheduler/cache/commonstores"
 	nodestore "github.com/kubewharf/godel-scheduler/pkg/scheduler/cache/commonstores/node_store"
+	"github.com/kubewharf/godel-scheduler/pkg/util/generationstore"
 )
 
 // Snapshot is a snapshot of s NodeInfo and NodeTree order. The scheduler takes a
@@ -62,6 +63,10 @@ func NewEmptySnapshot(handler commoncache.CacheHandler) *Snapshot {
 	return s
 }
 
+func (s *Snapshot) GetNodeStoreGeneration() int64 {
+	return s.CommonStoresSwitch.Find(nodestore.Name).(*nodestore.NodeStore).Store.(generationstore.RawStore).GetGeneration()
+}
+
 func (s *Snapshot) MakeBasicNodeGroup() framework.NodeGroup {
 	nodeCircle := framework.NewNodeCircle(framework.DefaultNodeCircleName, s)
 	nodeGroup := framework.NewNodeGroup(framework.DefaultNodeGroupName, s, []framework.NodeCircle{nodeCircle})