build: sync with latest GIE v0.0.0-20250715021823 (#239)

Update the inference scheduler to the latest GIE version- 20250715021823.

Signed-off-by: Kfir Toledo <[email protected]>

Author: kfirtoledo

deploy/config/epp-config.yaml

@@ -3,18 +3,18 @@
 apiVersion: inference.networking.x-k8s.io/v1alpha1
 kind: EndpointPickerConfig
 plugins:
-- type: prefix-cache
+- type: prefix-cache-scorer
   parameters:
     hashBlockSize: 5
     maxPrefixBlocksToMatch: 256
     lruCapacityPerServer: 31250
 - type: decode-filter
-- type: max-score
-- type: single-profile
+- type: max-score-picker
+- type: single-profile-handler
 schedulingProfiles:
 - name: default
   plugins:
   - pluginRef: decode-filter
-  - pluginRef: max-score
-  - pluginRef: prefix-cache
-    weight: 50
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
+    weight: 50

deploy/config/epp-kvcache-load-config.yaml

@@ -3,11 +3,11 @@
 apiVersion: inference.networking.x-k8s.io/v1alpha1
 kind: EndpointPickerConfig
 plugins:
-- type: single-profile
+- type: single-profile-handler
 - type: decode-filter
 - type: kvcache-aware-scorer
 - type: load-aware-scorer
-- type: max-score
+- type: max-score-picker
 schedulingProfiles:
 - name: default
   plugins:
@@ -16,4 +16,4 @@ schedulingProfiles:
     weight: 2.0
   - pluginRef: load-aware-scorer
     weight: 1.0
-  - pluginRef: max-score
+  - pluginRef: max-score-picker

deploy/config/pd-epp-config.yaml

@@ -2,15 +2,15 @@
 apiVersion: inference.networking.x-k8s.io/v1alpha1
 kind: EndpointPickerConfig
 plugins:
-- type: prefill-header
-- type: prefix-cache
+- type: prefill-header-handler
+- type: prefix-cache-scorer
   parameters:
     hashBlockSize: 5
     maxPrefixBlocksToMatch: 256
     lruCapacityPerServer: 31250
 - type: prefill-filter
 - type: decode-filter
-- type: max-score
+- type: max-score-picker
 - type: pd-profile-handler
   parameters:
     threshold: 10
@@ -19,12 +19,12 @@ schedulingProfiles:
 - name: prefill
   plugins:
   - pluginRef: prefill-filter
-  - pluginRef: max-score
-  - pluginRef: prefix-cache
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
     weight: 50
 - name: decode
   plugins:
   - pluginRef: decode-filter
-  - pluginRef: max-score
-  - pluginRef: prefix-cache
-    weight: 50
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
+    weight: 50

docs/architecture.md

@@ -83,7 +83,7 @@ These components are maintained in the `llm-d-inference-scheduler` repository an
 
 ## Configuration
 
-The set of lifecycle hooks (plugins) that are used by the inference scheduler is determined by how 
+The set of lifecycle hooks (plugins) that are used by the inference scheduler is determined by how
 it is configured. The configuration is in the form of YAML text, which can either be in a file or
 specified in-line as a parameter. The configuration defines the set of plugins to be instantiated along with their parameters. Each plugin is also given a name, enabling the same plugin type to be instantiated
 multiple times, if needed. Also defined is a set of SchedulingProfiles, which determine the set of
@@ -144,20 +144,20 @@ A complete configuration might look like this:
 apiVersion: inference.networking.x-k8s.io/v1alpha1
 kind: EndpointPickerConfig
 plugins:
-- type: prefix-cache
+- type: prefix-cache-scorer
   parameters:
     hashBlockSize: 5
     maxPrefixBlocksToMatch: 256
     lruCapacityPerServer: 31250
 - type: decode-filter
-- type: max-score
-- type: single-profile
+- type: max-score-picker
+- type: single-profile-handler
 schedulingProfiles:
 - name: default
   plugins:
   - pluginRef: decode-filter
-  - pluginRef: max-score
-  - pluginRef: prefix-cache
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
     weight: 50
 ```
 
@@ -170,9 +170,9 @@ This section describes how to setup the various plugins available with the llm-d
 
 **PrefillHeader**<br>
 Sets a header for use in disaggregated prefill/decode<br>
-*Type*: prefill-header<br>
+*Type*: prefill-header-handler<br>
 *Parameters*:<br>
-\- `prefillProfile` specifies the name of the profile used for the prefill scheduling. Only needed if the 
+\- `prefillProfile` specifies the name of the profile used for the prefill scheduling. Only needed if the
    prefill profile is not named `prefill`.<br>
 
 **PdProfileHandler**<br>
@@ -244,15 +244,15 @@ The following is an example of what a configuration for disaggregated Prefill/De
 apiVersion: inference.networking.x-k8s.io/v1alpha1
 kind: EndpointPickerConfig
 plugins:
-- type: prefill-header
-- type: prefix-cache
+- type: prefill-header-handler
+- type: prefix-cache-scorer
   parameters:
     hashBlockSize: 5
     maxPrefixBlocksToMatch: 256
     lruCapacityPerServer: 31250
 - type: prefill-filter
 - type: decode-filter
-- type: max-score
+- type: max-score-picker
 - type: pd-profile-handler
   parameters:
     threshold: 10
@@ -261,14 +261,14 @@ schedulingProfiles:
 - name: prefill
   plugins:
   - pluginRef: prefill-filter
-  - pluginRef: max-score
-  - pluginRef: prefix-cache
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
     weight: 50
 - name: decode
   plugins:
   - pluginRef: decode-filter
-  - pluginRef: max-score
-  - pluginRef: prefix-cache
+  - pluginRef: max-score-picker
+  - pluginRef: prefix-cache-scorer
     weight: 50
 ```
 

docs/dp.md

@@ -50,7 +50,7 @@ This evolved version removes the requirement for sidecars on the **prefill node*
      - If `prefill_worker_id == nil`, runs both stages locally by passing request to local vllm
      - If split:
        - Sends prefill job to Prefill Worker with a special header `do_remote_decode=true`
-       - Upon receiving response from Prefill Worker runs decode stage 
+       - Upon receiving response from Prefill Worker runs decode stage
 
 4. **Response Flow**
    - Response flows from decode sidecar → Envoy → EPP → User
@@ -119,7 +119,7 @@ This evolved version removes the requirement for sidecars on the **prefill node*
 
 ---
 
-## Diagram 
+## Diagram
 
 ![Disaggregated Prefill/Decode Architecture](./images/dp_architecture.png)
 

go.mod

@@ -14,7 +14,7 @@ require (
 	k8s.io/client-go v0.33.2
 	sigs.k8s.io/controller-runtime v0.21.0
 	sigs.k8s.io/gateway-api v1.3.0
-	sigs.k8s.io/gateway-api-inference-extension v0.0.0-20250704222130-0e1e964b9bec
+	sigs.k8s.io/gateway-api-inference-extension v0.0.0-20250715021823-e696e686a175
 )
 
 require (
@@ -40,27 +40,20 @@ require (
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
-	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/cel-go v0.23.2 // indirect
 	github.com/google/gnostic-models v0.6.9 // indirect
-	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
-	github.com/onsi/ginkgo/v2 v2.23.4 // indirect
-	github.com/onsi/gomega v1.37.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
@@ -81,7 +74,6 @@ require (
 	go.opentelemetry.io/otel/sdk v1.35.0 // indirect
 	go.opentelemetry.io/otel/trace v1.35.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.4.0 // indirect
-	go.uber.org/automaxprocs v1.6.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
@@ -93,7 +85,6 @@ require (
 	golang.org/x/term v0.32.0 // indirect
 	golang.org/x/text v0.25.0 // indirect
 	golang.org/x/time v0.9.0 // indirect
-	golang.org/x/tools v0.31.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20250324211829-b45e905df463 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250428153025-10db94c68c34 // indirect

go.sum

@@ -2,8 +2,6 @@ cel.dev/expr v0.23.0 h1:wUb94w6OYQS4uXraxo9U+wUAs9jT47Xvl4iPgAwM2ss=
 cel.dev/expr v0.23.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
 github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
-github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
-github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
@@ -126,8 +124,6 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
-github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
 github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q=
 github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0=
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
@@ -281,8 +277,8 @@ sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytI
 sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM=
 sigs.k8s.io/gateway-api v1.3.0 h1:q6okN+/UKDATola4JY7zXzx40WO4VISk7i9DIfOvr9M=
 sigs.k8s.io/gateway-api v1.3.0/go.mod h1:d8NV8nJbaRbEKem+5IuxkL8gJGOZ+FJ+NvOIltV8gDk=
-sigs.k8s.io/gateway-api-inference-extension v0.0.0-20250704222130-0e1e964b9bec h1:46xYN9Y8YZ/QHHrh+bEG0kXh9KFBxl6GMRjp2tgKcqk=
-sigs.k8s.io/gateway-api-inference-extension v0.0.0-20250704222130-0e1e964b9bec/go.mod h1:lki0jx1qysZSZT4Ai2BxuAcpx6G8g5oBgOGuuJzjy/k=
+sigs.k8s.io/gateway-api-inference-extension v0.0.0-20250715021823-e696e686a175 h1:jKDXyBPAiESgnmirAuX15MeES8yaVj+KykZDhwyAyrk=
+sigs.k8s.io/gateway-api-inference-extension v0.0.0-20250715021823-e696e686a175/go.mod h1:lki0jx1qysZSZT4Ai2BxuAcpx6G8g5oBgOGuuJzjy/k=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8=
 sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo=
 sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=

pkg/plugins/pre-request/pd_prerequest.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	// PrefillHeaderHandlerType is the type of the PrefillHeaderHandler
-	PrefillHeaderHandlerType = "prefill-header"
+	PrefillHeaderHandlerType = "prefill-header-handler"
 	// prefillPodHeader is the header name used to indicate Prefill worker <ip:port>
 	prefillPodHeader = "x-prefiller-host-port"
 
@@ -74,6 +74,6 @@ func (p *PrefillHeaderHandler) PreRequest(_ context.Context, request *types.LLMR
 		return // prefill profile failed to run or we chose not to run it, no-op in this case
 	}
 
-	prefillHostPort := net.JoinHostPort(prefillProfileRunResult.TargetPod.GetPod().Address, strconv.Itoa(targetPort))
+	prefillHostPort := net.JoinHostPort(prefillProfileRunResult.TargetPods[0].GetPod().Address, strconv.Itoa(targetPort))
 	request.Headers[prefillPodHeader] = prefillHostPort // in the form of <ip:port>
 }

pkg/plugins/profile/pd_profile_handler.go

@@ -108,7 +108,7 @@ func (h *PdProfileHandler) Pick(ctx context.Context, cycleState *types.CycleStat
 	if err != nil {
 		log.FromContext(ctx).Error(err, "unable to read prefix state")
 	} else {
-		decodePod := profileResults[h.decodeProfile].TargetPod.GetPod().NamespacedName
+		decodePod := profileResults[h.decodeProfile].TargetPods[0].GetPod().NamespacedName
 		hitPrefix := max(prefixState.PrefixCacheServers[prefix.ServerID(decodePod)]-1, 0) // The first hit is always the model name
 		hitPercentagePrefix = float64(hitPrefix*h.hashBlockSize) / float64(len(request.Prompt))
 		log.FromContext(ctx).V(logutil.DEBUG).Info("Computed hit percentage for prefix cache", "hitPercentage", hitPercentagePrefix,

pkg/plugins/scorer/load_aware_scorer_test.go

@@ -28,6 +28,7 @@ func TestLoadBasedScorer(t *testing.T) {
 		{
 			name:   "load based scorer",
 			scorer: scorer.NewLoadAwareScorer(context.Background(), 10),
+
 			req: &types.LLMRequest{
 				TargetModel: "critical",
 			},
@@ -72,7 +73,7 @@ func TestLoadBasedScorer(t *testing.T) {
 				},
 			},
 			wantRes: &types.ProfileRunResult{
-				TargetPod: &types.ScoredPod{
+				TargetPods: []types.Pod{&types.ScoredPod{
 					Pod: &types.PodMetrics{
 						Pod: &backend.Pod{
 							NamespacedName: k8stypes.NamespacedName{Name: "pod2"},
@@ -89,6 +90,7 @@ func TestLoadBasedScorer(t *testing.T) {
 					},
 					Score: 0.5,
 				},
+				},
 			},
 		},
 	}
@@ -97,7 +99,7 @@ func TestLoadBasedScorer(t *testing.T) {
 		t.Run(test.name, func(t *testing.T) {
 			schedulerProfile := framework.NewSchedulerProfile().
 				WithScorers(framework.NewWeightedScorer(test.scorer, 1)).
-				WithPicker(picker.NewMaxScorePicker())
+				WithPicker(picker.NewMaxScorePicker(picker.DefaultMaxNumOfEndpoints))
 
 			got, err := schedulerProfile.Run(context.Background(), test.req, nil, test.input)