Add scorer dependencies to ensure SLO scorer has the requisit scores from other plugins

Author: BenjaminBraunDev

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/elastic/crd-ref-docs v0.2.0
 	github.com/envoyproxy/go-control-plane/envoy v1.35.0
 	github.com/go-logr/logr v1.4.3
+	github.com/go-logr/zapr v1.3.0
 	github.com/google/go-cmp v0.7.0
 	github.com/google/uuid v1.6.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
@@ -61,7 +62,6 @@ require (
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
-	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.2 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.1 // indirect

pkg/epp/scheduling/framework/plugins.go

@@ -61,6 +61,7 @@ type Filter interface {
 type Scorer interface {
 	plugins.Plugin
 	Score(ctx context.Context, cycleState *types.CycleState, request *types.LLMRequest, pods []types.Pod) map[types.Pod]float64
+	Dependencies() []plugins.TypedName
 }
 
 // Picker picks the final pod(s) to send the request to.

pkg/epp/scheduling/framework/plugins/multi/prefix/plugin.go

@@ -91,6 +91,11 @@ type Plugin struct {
 	wg          sync.WaitGroup
 }
 
+// Dependencies implements framework.Scorer.
+func (p *Plugin) Dependencies() []plugins.TypedName {
+	return []plugins.TypedName{} // No dependencies
+}
+
 // podSet holds an pods servers that may have a specific prefix hash.
 type podSet map[ServerID]struct{}
 

pkg/epp/scheduling/framework/plugins/multi/slo_aware_router/plugin.go

@@ -219,6 +219,12 @@ type SLOAwareRouter struct {
 	headroomStrategy    HeadroomStrategy
 }
 
+func (s *SLOAwareRouter) Dependencies() []plugins.TypedName {
+	return []plugins.TypedName{
+		{Type: "prefix-cache-scorer", Name: "prefix-cache-scorer"},
+	}
+}
+
 var _ framework.Scorer = &SLOAwareRouter{}
 
 func NewSLOAwareRouter(latencypredictor latencypredictor.PredictorInterface, strategy HeadroomStrategy) *SLOAwareRouter {

pkg/epp/scheduling/framework/plugins/scorer/kvcache_utilization.go

@@ -50,6 +50,10 @@ type KVCacheUtilizationScorer struct {
 	typedName plugins.TypedName
 }
 
+func (s *KVCacheUtilizationScorer) Dependencies() []plugins.TypedName {
+	return []plugins.TypedName{} // No dependencies
+}
+
 // TypedName returns the type and name tuple of this plugin instance.
 func (s *KVCacheUtilizationScorer) TypedName() plugins.TypedName {
 	return s.typedName

pkg/epp/scheduling/framework/plugins/scorer/lora_affinity.go

@@ -50,6 +50,10 @@ type LoraAffinityScorer struct {
 	tn plugins.TypedName
 }
 
+func (s *LoraAffinityScorer) Dependencies() []plugins.TypedName {
+	return []plugins.TypedName{} // No dependencies
+}
+
 // TypedName returns the type and name tuple of this plugin instance.
 func (s *LoraAffinityScorer) TypedName() plugins.TypedName {
 	return s.tn

pkg/epp/scheduling/framework/plugins/scorer/queue.go

@@ -52,6 +52,10 @@ type QueueScorer struct {
 	typedName plugins.TypedName
 }
 
+func (s *QueueScorer) Dependencies() []plugins.TypedName {
+	return []plugins.TypedName{} // No dependencies
+}
+
 // TypedName returns the type and name tuple of this plugin instance.
 func (s *QueueScorer) TypedName() plugins.TypedName {
 	return s.typedName

pkg/epp/scheduling/framework/scheduler_profile.go

@@ -151,18 +151,25 @@ func (p *SchedulerProfile) runScorerPlugins(ctx context.Context, request *types.
 	logger := log.FromContext(ctx)
 	logger.V(logutil.DEBUG).Info("Before running scorer plugins", "pods", pods)
 
+	sortedScorers, err := p.topologicalSortScorers()
+	if err != nil {
+		logger.Error(err, "Failed to resolve scorer dependencies")
+		// Fallback to original order if dependency resolution fails
+		sortedScorers = p.scorers
+	}
+
 	weightedScorePerPod := make(map[types.Pod]float64, len(pods))
 	for _, pod := range pods {
 		weightedScorePerPod[pod] = float64(0) // initialize weighted score per pod with 0 value
 	}
 	// Iterate through each scorer in the chain and accumulate the weighted scores.
-	for _, scorer := range p.scorers {
+	for _, scorer := range sortedScorers {
 		logger.V(logutil.DEBUG).Info("Running scorer plugin", "plugin", scorer.TypedName())
 		before := time.Now()
 		scores := scorer.Score(ctx, cycleState, request, pods)
 		metrics.RecordPluginProcessingLatency(ScorerExtensionPoint, scorer.TypedName().Type, scorer.TypedName().Name, time.Since(before))
 		for pod, score := range scores { // weight is relative to the sum of weights
-			logger.V(logutil.DEBUG).Info("Calculated score", "plugin", scorer.TypedName(), "endpoint", pod.GetPod().NamespacedName, "score", score)
+			logger.V(logutil.DEBUG).Info("Calculated score", "plugin", scorer.TypedName(), "endpoint", pod.GetPod().NamespacedName, "score", score, "weight", scorer.Weight())
 			weightedScorePerPod[pod] += enforceScoreRange(score) * float64(scorer.Weight())
 		}
 		logger.V(logutil.DEBUG).Info("Completed running scorer plugin successfully", "plugin", scorer.TypedName())
@@ -190,6 +197,77 @@ func (p *SchedulerProfile) runPickerPlugin(ctx context.Context, cycleState *type
 	return result
 }
 
+func (p *SchedulerProfile) topologicalSortScorers() ([]*WeightedScorer, error) {
+	if len(p.scorers) == 0 {
+		return p.scorers, nil
+	}
+
+	// Create maps for efficient lookups
+	scorerByName := make(map[string]*WeightedScorer)
+	inDegree := make(map[string]int)
+	adjList := make(map[string][]string)
+
+	// Initialize data structures
+	for _, scorer := range p.scorers {
+		name := scorer.TypedName().String()
+		scorerByName[name] = scorer
+		inDegree[name] = 0
+		adjList[name] = []string{}
+	}
+
+	// Build adjacency list and calculate in-degrees
+	for _, scorer := range p.scorers {
+		scorerName := scorer.TypedName().String()
+		for _, dep := range scorer.Dependencies() {
+			depName := dep.String()
+
+			// Check if dependency exists in our scorer list
+			if _, exists := scorerByName[depName]; !exists {
+				return nil, fmt.Errorf("scorer '%s' depends on '%s' which is not registered in the profile", scorerName, depName)
+			}
+
+			// Add edge: dependency -> dependent
+			adjList[depName] = append(adjList[depName], scorerName)
+			inDegree[scorerName]++
+		}
+	}
+
+	// Kahn's algorithm for topological sorting
+	var queue []string
+	var result []*WeightedScorer
+
+	// Find all nodes with no incoming edges
+	for name, degree := range inDegree {
+		if degree == 0 {
+			queue = append(queue, name)
+		}
+	}
+
+	for len(queue) > 0 {
+		// Remove a node from queue
+		current := queue[0]
+		queue = queue[1:]
+
+		// Add to result
+		result = append(result, scorerByName[current])
+
+		// For each neighbor of current node
+		for _, neighbor := range adjList[current] {
+			inDegree[neighbor]--
+			if inDegree[neighbor] == 0 {
+				queue = append(queue, neighbor)
+			}
+		}
+	}
+
+	// Check for cycles
+	if len(result) != len(p.scorers) {
+		return nil, fmt.Errorf("circular dependency detected in scorer plugins")
+	}
+
+	return result, nil
+}
+
 func enforceScoreRange(score float64) float64 {
 	if score < 0 {
 		return 0

pkg/epp/scheduling/framework/scheduler_profile_test.go

@@ -144,7 +144,7 @@ func TestSchedulePlugins(t *testing.T) {
 				},
 			}
 
-			if diff := cmp.Diff(wantRes, got); diff != "" {
+			if diff := cmp.Diff(wantRes, got, podScoresTransformer); diff != "" {
 				t.Errorf("Unexpected output (-want +got): %v", diff)
 			}
 			// Validate plugin execution counts dynamically
@@ -182,6 +182,21 @@ var _ Filter = &testPlugin{}
 var _ Scorer = &testPlugin{}
 var _ Picker = &testPlugin{}
 
+// podScoresTransformer converts a map keyed by types.Pod into a map keyed by
+// the pod's unique name string. This allows cmp.Diff to compare the maps based
+// on their semantic content rather than on pointer addresses.
+var podScoresTransformer = cmp.Transformer("podScores", func(in map[types.Pod]float64) map[string]float64 {
+	out := make(map[string]float64, len(in))
+	if in == nil {
+		return nil
+	}
+	for pod, score := range in {
+		// Use the pod's unique NamespacedName as the stable key
+		out[pod.GetPod().NamespacedName.String()] = score
+	}
+	return out
+})
+
 // testPlugin is an implementation useful in unit tests.
 type testPlugin struct {
 	typedName             plugins.TypedName
@@ -197,6 +212,11 @@ type testPlugin struct {
 	WinnerPodScore        float64
 }
 
+// Dependencies implements Scorer.
+func (tp *testPlugin) Dependencies() []plugins.TypedName {
+	return []plugins.TypedName{} // No dependencies
+}
+
 func (tp *testPlugin) TypedName() plugins.TypedName {
 	return tp.typedName
 }

pkg/epp/scheduling/framework/weighted_scorer.go

@@ -16,21 +16,29 @@ limitations under the License.
 
 package framework
 
+import "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
+
 // NewWeightedScorer initializes a new WeightedScorer and returns its pointer.
 func NewWeightedScorer(scorer Scorer, weight int) *WeightedScorer {
 	return &WeightedScorer{
-		Scorer: scorer,
-		weight: weight,
+		Scorer:       scorer,
+		weight:       weight,
+		dependencies: scorer.Dependencies(),
 	}
 }
 
 // WeightedScorer is a struct that encapsulates a scorer with its weight.
 type WeightedScorer struct {
 	Scorer
-	weight int
+	weight       int
+	dependencies []plugins.TypedName
 }
 
 // Weight returns the weight of the scorer.
 func (s *WeightedScorer) Weight() int {
 	return s.weight
 }
+
+func (ws *WeightedScorer) Dependencies() []plugins.TypedName {
+	return ws.dependencies
+}