load plugins from file in conformance tests (#1152)

* configure conformance plugins through file, remove dep on conformance

Signed-off-by: Nir Rozenbaum <[email protected]>

* update config filename to conformance-plugins.yaml

Signed-off-by: Nir Rozenbaum <[email protected]>

---------

Signed-off-by: Nir Rozenbaum <[email protected]>

Author: nirrozenbaum

Dockerfile

@@ -19,7 +19,6 @@ RUN go mod download
 # Sources
 COPY cmd/epp ./cmd/epp
 COPY pkg/epp ./pkg/epp
-COPY conformance/testing-epp ./conformance/testing-epp
 COPY internal ./internal
 COPY api ./api
 WORKDIR /src/cmd/epp

cmd/epp/runner/register.go

@@ -25,6 +25,7 @@ import (
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/picker"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/profile"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/scorer"
+	testfilter "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework/plugins/test/filter"
 )
 
 // RegisterAllPlugins registers the factory functions of all known plugins
@@ -40,6 +41,8 @@ func RegisterAllPlugins() {
 	plugins.Register(profile.SingleProfileHandlerType, profile.SingleProfileHandlerFactory)
 	plugins.Register(scorer.KvCacheScorerType, scorer.KvCacheScorerFactory)
 	plugins.Register(scorer.QueueScorerType, scorer.QueueScorerFactory)
+	// register filter for test purpose only (used in conformance tests)
+	plugins.Register(testfilter.HeaderBasedTestingFilterType, testfilter.HeaderBasedTestingFilterFactory)
 }
 
 // eppHandle is an implementation of the interface plugins.Handle

cmd/epp/runner/runner.go

@@ -37,7 +37,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/metrics/filters"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
-	conformance_epp "sigs.k8s.io/gateway-api-inference-extension/conformance/testing-epp"
 	"sigs.k8s.io/gateway-api-inference-extension/internal/runnable"
 	backendmetrics "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend/metrics"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/common/config/loader"
@@ -48,7 +47,6 @@ import (
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/saturationdetector"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling"
 	runserver "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/server"
-	envutil "sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/env"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/util/logging"
 )
 
@@ -141,9 +139,6 @@ var (
 	modelServerMetricsPath = flag.String("modelServerMetricsPath", "/metrics", "Path to scrape metrics from pods")
 
 	setupLog = ctrl.Log.WithName("setup")
-
-	// Environment variables
-	reqHeaderBasedSchedulerForTesting = envutil.GetEnvBool("ENABLE_REQ_HEADER_BASED_SCHEDULER_FOR_TESTING", false, setupLog)
 )
 
 // NewRunner initializes a new EPP Runner and returns its pointer.
@@ -337,13 +332,7 @@ func (r *Runner) initializeScheduler() *scheduling.Scheduler {
 	}
 
 	// otherwise, no one configured from outside scheduler config. use existing configuration
-	scheduler := scheduling.NewScheduler()
-
-	if reqHeaderBasedSchedulerForTesting {
-		scheduler = conformance_epp.NewReqHeaderBasedScheduler()
-	}
-
-	return scheduler
+	return scheduling.NewScheduler()
 }
 
 func (r *Runner) parseConfiguration(ctx context.Context) error {

conformance/resources/manifests/manifests.yaml

@@ -211,11 +211,11 @@ spec:
         - "9002"
         - -grpcHealthPort
         - "9003"
+        - "-configFile"
+        - "/config/conformance-plugins.yaml"
         env:
         - name: USE_STREAMING
           value: "true"
-        - name: ENABLE_REQ_HEADER_BASED_SCHEDULER_FOR_TESTING # Used for conformance test.
-          value: "true"
         ports:
         - containerPort: 9002
         - containerPort: 9003
@@ -233,6 +233,13 @@ spec:
             service: inference-extension
           initialDelaySeconds: 5
           periodSeconds: 10
+        volumeMounts:
+        - name: plugins-config-volume
+          mountPath: "/config"
+      volumes:
+      - name: plugins-config-volume
+        configMap:
+          name: plugins-config
 ---
 # --- Secondary InferencePool Definition ---
 apiVersion: inference.networking.x-k8s.io/v1alpha2
@@ -301,11 +308,11 @@ spec:
         - "9002"
         - -grpcHealthPort
         - "9003"
+        - "-configFile"
+        - "/config/conformance-plugins.yaml"
         env:
         - name: USE_STREAMING
           value: "true"
-        - name: ENABLE_REQ_HEADER_BASED_SCHEDULER_FOR_TESTING # Used for conformance test.
-          value: "true"
         ports:
         - containerPort: 9002
         - containerPort: 9003
@@ -323,6 +330,34 @@ spec:
             service: inference-extension
           initialDelaySeconds: 5
           periodSeconds: 10
+        volumeMounts:
+        - name: plugins-config-volume
+          mountPath: "/config"
+      volumes:
+      - name: plugins-config-volume
+        configMap:
+          name: plugins-config
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: plugins-config
+  namespace: default
+data:
+  conformance-plugins.yaml: |
+    apiVersion: inference.networking.x-k8s.io/v1alpha1
+    kind: EndpointPickerConfig
+    plugins:
+    - type: header-based-testing-filter
+    - type: max-score-picker
+      parameters:
+        maxNumOfEndpoints: 1
+    - type: single-profile-handler
+    schedulingProfiles:
+    - name: conformance-profile
+      plugins:
+      - pluginRef: header-based-testing-filter
+      - pluginRef: max-score-picker
 ---
 # --- Required Role and RoleBinding for Conformance Test for EPP ---
 apiVersion: rbac.authorization.k8s.io/v1

conformance/testing-epp/scheduler.go

@@ -0,0 +1 @@
+This package contains plugins implementation for test purpose only.

conformance/testing-epp/scheduler_test.go

@@ -22,22 +22,19 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/backend"
-	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/framework"
 	"sigs.k8s.io/gateway-api-inference-extension/pkg/epp/scheduling/types"
 )
 
 func TestFilter(t *testing.T) {
 	tests := []struct {
 		name   string
 		req    *types.LLMRequest
-		filter framework.Filter
 		input  []types.Pod
 		output []types.Pod
 	}{
 		{
-			name:   "TestHeaderBasedFilter, header endpoint unset in request",
-			req:    &types.LLMRequest{}, // Delieverately unset the header.
-			filter: &HeaderBasedTestingFilter{},
+			name: "TestHeaderBasedFilter, header endpoint unset in request",
+			req:  &types.LLMRequest{}, // Delieverately unset the header.
 			input: []types.Pod{
 				&types.PodMetrics{
 					Pod: &backend.Pod{
@@ -48,9 +45,8 @@ func TestFilter(t *testing.T) {
 			output: []types.Pod{},
 		},
 		{
-			name:   "TestHeaderBasedFilter, header endpoint set in request but no match",
-			req:    &types.LLMRequest{Headers: map[string]string{headerTestEppEndPointSelectionKey: "test-endpoint"}},
-			filter: &HeaderBasedTestingFilter{},
+			name: "TestHeaderBasedFilter, header endpoint set in request but no match",
+			req:  &types.LLMRequest{Headers: map[string]string{headerTestEppEndPointSelectionKey: "test-endpoint"}},
 			input: []types.Pod{
 				&types.PodMetrics{
 					Pod: &backend.Pod{
@@ -61,9 +57,8 @@ func TestFilter(t *testing.T) {
 			output: []types.Pod{},
 		},
 		{
-			name:   "TestHeaderBasedFilter, header endpoint set",
-			req:    &types.LLMRequest{Headers: map[string]string{headerTestEppEndPointSelectionKey: "test-endpoint"}},
-			filter: &HeaderBasedTestingFilter{},
+			name: "TestHeaderBasedFilter, header endpoint set",
+			req:  &types.LLMRequest{Headers: map[string]string{headerTestEppEndPointSelectionKey: "test-endpoint"}},
 			input: []types.Pod{
 				&types.PodMetrics{
 					Pod: &backend.Pod{
@@ -80,9 +75,8 @@ func TestFilter(t *testing.T) {
 			},
 		},
 		{
-			name:   "TestHeaderBasedFilter, multiple header endpoints set and multiple matches",
-			req:    &types.LLMRequest{Headers: map[string]string{headerTestEppEndPointSelectionKey: "test-endpoint3,test-endpoint2"}},
-			filter: &HeaderBasedTestingFilter{},
+			name: "TestHeaderBasedFilter, multiple header endpoints set and multiple matches",
+			req:  &types.LLMRequest{Headers: map[string]string{headerTestEppEndPointSelectionKey: "test-endpoint3,test-endpoint2"}},
 			input: []types.Pod{
 				&types.PodMetrics{
 					Pod: &backend.Pod{
@@ -117,7 +111,7 @@ func TestFilter(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			got := test.filter.Filter(context.Background(), types.NewCycleState(), test.req, test.input)
+			got := NewHeaderBasedTestingFilter().Filter(context.Background(), types.NewCycleState(), test.req, test.input)
 
 			if diff := cmp.Diff(test.output, got); diff != "" {
 				t.Errorf("Unexpected output (-want +got): %v", diff)