Fix: Merge Extra field during VirtualService merge for multiple Infer…

pilot/pkg/config/kube/gateway/conversion_test.go

@@ -158,6 +158,30 @@ var services = []*model.Service{
 		Ports:    inferencePoolPorts,
 		Hostname: host.Name(fmt.Sprintf("%s.default.svc.domain.suffix", firstValue(InferencePoolServiceName("infpool-gen2")))),
 	},
+	{
+		Attributes: model.ServiceAttributes{
+			Namespace: "default",
+			Labels: map[string]string{
+				InferencePoolExtensionRefSvc:         "model1-epp",
+				InferencePoolExtensionRefPort:        "9002",
+				InferencePoolExtensionRefFailureMode: "FailClose",
+			},
+		},
+		Ports:    ports,
+		Hostname: host.Name(fmt.Sprintf("%s.default.svc.domain.suffix", firstValue(InferencePoolServiceName("infpool-model1")))),
+	},
+	{
+		Attributes: model.ServiceAttributes{
+			Namespace: "default",
+			Labels: map[string]string{
+				InferencePoolExtensionRefSvc:         "model2-epp",
+				InferencePoolExtensionRefPort:        "9002",
+				InferencePoolExtensionRefFailureMode: "FailClose",
+			},
+		},
+		Ports:    ports,
+		Hostname: host.Name(fmt.Sprintf("%s.default.svc.domain.suffix", firstValue(InferencePoolServiceName("infpool-model2")))),
+	},
 
 	{
 		Attributes: model.ServiceAttributes{

pilot/pkg/config/kube/gateway/route_collections.go

@@ -682,12 +682,63 @@ func mergeHTTPRoutes(baseVirtualServices krt.Collection[RouteWithKey], opts ...k
 		sortRoutesByCreationTime(configs)
 		base := configs[0].DeepCopy()
 		baseVS := base.Spec.(*istio.VirtualService)
-		for _, config := range configs[1:] {
+		// Deep copy the InferencePool configs map to avoid race conditions
+		// The default DeepCopy() only does shallow copy of Extra field
+		if base.Extra != nil {
+			if ipConfigs, ok := base.Extra[constants.ConfigExtraPerRouteRuleInferencePoolConfigs].(map[string]kube.InferencePoolRouteRuleConfig); ok {
+				// Create a new map to avoid modifying the shared underlying map
+				newIPConfigs := make(map[string]kube.InferencePoolRouteRuleConfig, len(ipConfigs))
+				for k, v := range ipConfigs {
+					newIPConfigs[k] = v
+				}
+				base.Extra[constants.ConfigExtraPerRouteRuleInferencePoolConfigs] = newIPConfigs
+			}
+		}
+		for i, config := range configs[1:] {
 			thisVS := config.Spec.(*istio.VirtualService)
 			baseVS.Http = append(baseVS.Http, thisVS.Http...)
 			// append parents
 			base.Annotations[constants.InternalParentNames] = fmt.Sprintf("%s,%s",
 				base.Annotations[constants.InternalParentNames], config.Annotations[constants.InternalParentNames])
+			// Merge Extra field (especially for InferencePool configs)
+			if base.Extra == nil && config.Extra != nil {
+				base.Extra = make(map[string]any)
+			}
+			if config.Extra != nil {
+				for k, v := range config.Extra {
+					// For non-InferencePool configs, keep the first value for stability
+					if k != constants.ConfigExtraPerRouteRuleInferencePoolConfigs {
+						if _, exists := base.Extra[k]; !exists {
+							base.Extra[k] = v
+						}
+						continue
+					}
+					// For InferencePool configs, merge the maps
+					baseMap, baseOk := base.Extra[k].(map[string]kube.InferencePoolRouteRuleConfig)
+					configMap, configOk := v.(map[string]kube.InferencePoolRouteRuleConfig)
+					if baseOk && configOk {
+						log.Debugf("Merging InferencePool configs: adding %d route configs from VirtualService %d to base (namespace=%s)",
+							len(configMap), i+1, config.Namespace)
+						// Route names are composed of the HTTPRoute/VirtualService namespaced name so they can't possibly conflict
+						for routeName, routeConfig := range configMap {
+							baseMap[routeName] = routeConfig
+						}
+					} else if configOk {
+						if _, exists := base.Extra[k]; !exists {
+							log.Debugf("Creating new InferencePool config map from VirtualService %d (namespace=%s)", i+1, config.Namespace)
+							base.Extra[k] = v
+						}
+					} else if !configOk {
+						log.Debugf("Skipping InferencePool config from VirtualService %d due to unexpected type (namespace=%s)", i+1, config.Namespace)
+					}
+				}
+			}
+		}
+		// Log final merged InferencePool configs
+		if base.Extra != nil {
+			if ipConfigs, ok := base.Extra[constants.ConfigExtraPerRouteRuleInferencePoolConfigs].(map[string]kube.InferencePoolRouteRuleConfig); ok {
+				log.Debugf("Final merged VirtualService for key %s has %d InferencePool route configs", object.Key, len(ipConfigs))
+			}
 		}
 		sortHTTPRoutes(baseVS.Http)
 		base.Name = strings.ReplaceAll(object.Key, "/", "~")

pilot/pkg/config/kube/gateway/testdata/http.status.yaml.golden

@@ -16,6 +16,24 @@ metadata:
 spec: null
 status: {}
 ---
+apiVersion: inference.networking.k8s.io/v1
+kind: InferencePool
+metadata:
+  creationTimestamp: null
+  name: infpool-model1
+  namespace: default
+spec: null
+status: {}
+---
+apiVersion: inference.networking.k8s.io/v1
+kind: InferencePool
+metadata:
+  creationTimestamp: null
+  name: infpool-model2
+  namespace: default
+spec: null
+status: {}
+---
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: GatewayClass
 metadata:
@@ -53,7 +71,7 @@ status:
     status: "True"
     type: Programmed
   listeners:
-  - attachedRoutes: 11
+  - attachedRoutes: 13
     conditions:
     - lastTransitionTime: fake
       message: No errors found
@@ -284,6 +302,56 @@ status:
 ---
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: HTTPRoute
+metadata:
+  creationTimestamp: null
+  name: multi-route-infpool-1
+  namespace: default
+spec: null
+status:
+  parents:
+  - conditions:
+    - lastTransitionTime: fake
+      message: Route was valid
+      reason: Accepted
+      status: "True"
+      type: Accepted
+    - lastTransitionTime: fake
+      message: All references resolved
+      reason: ResolvedRefs
+      status: "True"
+      type: ResolvedRefs
+    controllerName: istio.io/gateway-controller
+    parentRef:
+      name: gateway
+      namespace: istio-system
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  creationTimestamp: null
+  name: multi-route-infpool-2
+  namespace: default
+spec: null
+status:
+  parents:
+  - conditions:
+    - lastTransitionTime: fake
+      message: Route was valid
+      reason: Accepted
+      status: "True"
+      type: Accepted
+    - lastTransitionTime: fake
+      message: All references resolved
+      reason: ResolvedRefs
+      status: "True"
+      type: ResolvedRefs
+    controllerName: istio.io/gateway-controller
+    parentRef:
+      name: gateway
+      namespace: istio-system
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
 metadata:
   creationTimestamp: null
   name: multiple-inferencepool-backend-refs

pilot/pkg/config/kube/gateway/testdata/http.yaml

@@ -423,3 +423,79 @@ spec:
     name: vllm-llama3-8b-instruct-epp
     port:
       number: 9002
+---
+# Test case for multiple HTTPRoutes with InferencePools on same gateway
+# This tests that Extra field (InferencePool configs) are properly merged
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: multi-route-infpool-1
+  namespace: default
+spec:
+  parentRefs:
+    - name: gateway
+      namespace: istio-system
+  hostnames: ["multi-infpool.domain.example"]
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /model1
+      backendRefs:
+        - name: infpool-model1
+          group: inference.networking.k8s.io
+          kind: InferencePool
+          port: 80
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: HTTPRoute
+metadata:
+  name: multi-route-infpool-2
+  namespace: default
+spec:
+  parentRefs:
+    - name: gateway
+      namespace: istio-system
+  hostnames: ["multi-infpool.domain.example"]
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /model2
+      backendRefs:
+        - name: infpool-model2
+          group: inference.networking.k8s.io
+          kind: InferencePool
+          port: 80
+---
+apiVersion: inference.networking.k8s.io/v1
+kind: InferencePool
+metadata:
+  name: infpool-model1
+  namespace: default
+spec:
+  targetPorts:
+  - number: 8000
+  selector:
+    matchLabels:
+      app: model1-server
+  endpointPickerRef:
+    name: model1-epp
+    port:
+      number: 9002
+---
+apiVersion: inference.networking.k8s.io/v1
+kind: InferencePool
+metadata:
+  name: infpool-model2
+  namespace: default
+spec:
+  targetPorts:
+  - number: 8000
+  selector:
+    matchLabels:
+      app: model2-server
+  endpointPickerRef:
+    name: model2-epp
+    port:
+      number: 9002

pilot/pkg/config/kube/gateway/testdata/http.yaml.golden

@@ -252,6 +252,40 @@ spec:
 ---
 apiVersion: networking.istio.io/v1
 kind: VirtualService
+metadata:
+  annotations:
+    internal.istio.io/parents: HTTPRoute/multi-route-infpool-1.default,HTTPRoute/multi-route-infpool-2.default
+    internal.istio.io/route-semantics: gateway
+  creationTimestamp: null
+  name: istio-system~gateway-istio-autogenerated-k8s-gateway-default~multi-infpool.domain.example
+  namespace: default
+spec:
+  gateways:
+  - istio-system/gateway-istio-autogenerated-k8s-gateway-default
+  hosts:
+  - multi-infpool.domain.example
+  http:
+  - match:
+    - uri:
+        prefix: /model1
+    name: default.multi-route-infpool-1.0
+    route:
+    - destination:
+        host: infpool-model1-ip-aaaaf2d6.default.svc.domain.suffix
+        port:
+          number: 80
+  - match:
+    - uri:
+        prefix: /model2
+    name: default.multi-route-infpool-2.0
+    route:
+    - destination:
+        host: infpool-model2-ip-f857bff9.default.svc.domain.suffix
+        port:
+          number: 80
+---
+apiVersion: networking.istio.io/v1
+kind: VirtualService
 metadata:
   annotations:
     internal.istio.io/parents: HTTPRoute/redirect-prefix-replace.default

releasenotes/notes/58393.yaml

@@ -0,0 +1,9 @@
+apiVersion: release-notes/v2
+kind: bug-fix
+area: traffic-management
+issue:
+- 58392
+
+releaseNotes:
+- |
+  **Fixed** an issue where InferencePool configurations were lost during VirtualService merging when multiple HTTPRoutes referencing different InferencePools were attached to the same Gateway.