cleanup of config from scheduling package (#1263)

Signed-off-by: Nir Rozenbaum <[email protected]>

Author: nirrozenbaum

cmd/epp/runner/runner.go

@@ -42,7 +42,7 @@ import (
 
 	"sigs.k8s.io/gateway-api-inference-extension/internal/runnable"
 	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/common/config/loader"
+	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/config/loader"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/datastore"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/metrics/collectors"

pkg/epp/common/config/defaults.go

@@ -26,14 +26,19 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	configapi "sigs.k8s.io/gateway-api-inference-extension/apix/config/v1alpha1"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/common/config"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/picker"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/profile"
 )
 
+const (
+	// DefaultScorerWeight is the weight used for scorers referenced in the
+	// configuration without explicit weights.
+	DefaultScorerWeight = 1
+)
+
 var scheme = runtime.NewScheme()
 
 func init() {
@@ -72,7 +77,7 @@ func LoadSchedulerConfig(configProfiles []configapi.SchedulingProfile, handle pl
 			referencedPlugin := handle.Plugin(plugin.PluginRef)
 			if scorer, ok := referencedPlugin.(framework.Scorer); ok {
 				// Set default weight if one wasn't set in the configuration
-				weight := config.DefaultScorerWeight
+				weight := DefaultScorerWeight
 				if plugin.Weight != nil {
 					weight = *plugin.Weight
 				}

pkg/epp/common/config/loader/configloader.go renamed to pkg/epp/config/loader/configloader.go

@@ -193,7 +193,7 @@ func TestLoadConfiguration(t *testing.T) {
 		{
 			name:       "successFromFile",
 			configText: "",
-			configFile: "../../../../../test/testdata/configloader_1_test.yaml",
+			configFile: "../../../../test/testdata/configloader_1_test.yaml",
 			want:       goodConfig,
 			wantErr:    false,
 		},

pkg/epp/common/config/loader/configloader_test.go renamed to pkg/epp/config/loader/configloader_test.go

@@ -21,14 +21,15 @@ import (
 
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
-	commonconfig "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/common/config"
 	envutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/env"
 )
 
 // Default configuration values
 const (
-	DefaultQueueDepthThreshold  = commonconfig.DefaultQueueThresholdCritical
-	DefaultKVCacheUtilThreshold = commonconfig.DefaultKVCacheThreshold
+	// DefaultQueueDepthThreshold is the default backend waiting queue size threshold.
+	DefaultQueueDepthThreshold = 5
+	// DefaultKVCacheUtilThreshold is the default KV cache utilization (0.0 to 1.0) threshold.
+	DefaultKVCacheUtilThreshold = 0.8
 	// DefaultMetricsStalenessThreshold defines how old metrics can be before they
 	// are considered stale.
 	// Given the pod metrics refresh interval is 50ms, a threshold slightly above

pkg/epp/saturationdetector/config.go

@@ -29,7 +29,6 @@ import (
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend"
 	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/config"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/scorer"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
@@ -171,17 +170,8 @@ func TestLoRASoftAffinityDistribution(t *testing.T) {
 		tolerancePercent  = 5.0 // Allow 5% tolerance from expected distribution
 	)
 
-	// Save original config value to restore later
-	originalThreshold := config.Conf.LoraAffinityThreshold
-
 	// Set a specific test value for this test
 	testThreshold := 0.75 // 75%
-	config.Conf.LoraAffinityThreshold = testThreshold
-
-	// Ensure we restore the original threshold when test completes
-	defer func() {
-		config.Conf.LoraAffinityThreshold = originalThreshold
-	}()
 
 	// Create a test request and pods
 	req := &types.LLMRequest{
@@ -213,11 +203,11 @@ func TestLoRASoftAffinityDistribution(t *testing.T) {
 	availableCount := 0
 
 	// Use the test threshold value
-	expectedAffinityPercent := config.Conf.LoraAffinityThreshold * 100
+	expectedAffinityPercent := testThreshold * 100
 	expectedAvailabilityPercent := 100 - expectedAffinityPercent
 
 	// initialize LoraAffinityFilter
-	LoraAffinityFilter := NewLoraAffinityFilter(config.Conf.LoraAffinityThreshold)
+	LoraAffinityFilter := NewLoraAffinityFilter(testThreshold)
 
 	for range numIterations {
 		result := LoraAffinityFilter.Filter(context.Background(), types.NewCycleState(), req, pods)
@@ -247,8 +237,8 @@ func TestLoRASoftAffinityDistribution(t *testing.T) {
 	availableUpperBound := expectedAvailabilityPercent + tolerancePercent
 
 	t.Logf("Distribution results over %d iterations:", numIterations)
-	t.Logf("Expected affinity percent: %.2f%% (threshold: %.2f)", expectedAffinityPercent, config.Conf.LoraAffinityThreshold)
-	t.Logf("Expected availability percent: %.2f%% (threshold: %.2f)", expectedAvailabilityPercent, config.Conf.LoraAffinityThreshold)
+	t.Logf("Expected affinity percent: %.2f%% (threshold: %.2f)", expectedAffinityPercent, testThreshold)
+	t.Logf("Expected availability percent: %.2f%% (threshold: %.2f)", expectedAvailabilityPercent, testThreshold)
 	t.Logf("Actual affinity percent: %.2f%% (%d out of %d)", actualAffinityPercent, affinityCount, numIterations)
 	t.Logf("Actual available percent: %.2f%% (%d out of %d)", actualAvailablePercent, availableCount, numIterations)
 
@@ -268,8 +258,8 @@ func TestDecisionTreeFilterFactory(t *testing.T) {
 
 	leastKvCacheFilter := NewLeastKVCacheFilter()
 	leastQueueFilter := NewLeastQueueFilter()
-	loraAffinityFilter := NewLoraAffinityFilter(config.Conf.LoraAffinityThreshold)
-	lowQueueFilter := NewLowQueueFilter(config.Conf.QueueingThresholdLoRA)
+	loraAffinityFilter := NewLoraAffinityFilter(DefaultLoraAffinityThreshold)
+	lowQueueFilter := NewLowQueueFilter(DefaultQueueingThresholdLoRA)
 
 	kvCacheScorer := scorer.NewKVCacheUtilizationScorer()
 

pkg/epp/scheduling/config/config.go

@@ -24,13 +24,13 @@ import (
 	"time"
 
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/config"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
 )
 
 const (
-	LoraAffinityFilterType = "lora-affinity-filter"
+	LoraAffinityFilterType       = "lora-affinity-filter"
+	DefaultLoraAffinityThreshold = 0.999
 )
 
 type loraAffinityFilterParameters struct {
@@ -42,7 +42,7 @@ var _ framework.Filter = &LoraAffinityFilter{}
 
 // LoraAffinityFilterFactory defines the factory function for LoraAffinityFilter.
 func LoraAffinityFilterFactory(name string, rawParameters json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) {
-	parameters := loraAffinityFilterParameters{Threshold: config.DefaultLoraAffinityThreshold}
+	parameters := loraAffinityFilterParameters{Threshold: DefaultLoraAffinityThreshold}
 	if rawParameters != nil {
 		if err := json.Unmarshal(rawParameters, &parameters); err != nil {
 			return nil, fmt.Errorf("failed to parse the parameters of the '%s' filter - %w", LoraAffinityFilterType, err)

pkg/epp/scheduling/framework/plugins/filter/filter_test.go

@@ -22,13 +22,13 @@ import (
 	"fmt"
 
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/plugins"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/config"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
 )
 
 const (
-	LowQueueFilterType = "low-queue-filter"
+	LowQueueFilterType           = "low-queue-filter"
+	DefaultQueueingThresholdLoRA = 128
 )
 
 type lowQueueFilterParameters struct {
@@ -40,7 +40,7 @@ var _ framework.Filter = &LowQueueFilter{}
 
 // LowQueueFilterFactory defines the factory function for LowQueueFilter.
 func LowQueueFilterFactory(name string, rawParameters json.RawMessage, _ plugins.Handle) (plugins.Plugin, error) {
-	parameters := lowQueueFilterParameters{Threshold: config.DefaultQueueingThresholdLoRA}
+	parameters := lowQueueFilterParameters{Threshold: DefaultQueueingThresholdLoRA}
 	if rawParameters != nil {
 		if err := json.Unmarshal(rawParameters, &parameters); err != nil {
 			return nil, fmt.Errorf("failed to parse the parameters of the '%s' filter - %w", LowQueueFilterType, err)

pkg/epp/scheduling/framework/plugins/filter/lora_affinity_filter.go

@@ -26,7 +26,6 @@ import (
 
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend"
 	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics" // Import config for thresholds
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/config"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/filter"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/picker"
@@ -36,12 +35,12 @@ import (
 
 // Tests the default scheduler configuration and expected behavior.
 func TestSchedule(t *testing.T) {
-	loraAffinityFilter := filter.NewLoraAffinityFilter(config.Conf.LoraAffinityThreshold)
+	loraAffinityFilter := filter.NewLoraAffinityFilter(filter.DefaultLoraAffinityThreshold)
 	leastQueueFilter := filter.NewLeastQueueFilter()
 	leastKvCacheFilter := filter.NewLeastKVCacheFilter()
 
 	lowLatencyFilter := &filter.DecisionTreeFilter{
-		Current: filter.NewLowQueueFilter(config.Conf.QueueingThresholdLoRA),
+		Current: filter.NewLowQueueFilter(filter.DefaultQueueingThresholdLoRA),
 		NextOnSuccess: &filter.DecisionTreeFilter{
 			Current: loraAffinityFilter,
 			NextOnSuccessOrFailure: &filter.DecisionTreeFilter{