fix(webhook): relax HTTPRoute/Gateway validation (#911)

Users should be able to attach custom Gateway while still relying on
managed HTTPRoute. While controller already has the logic to support it,
the webhook was previously rejecting as invalid, expecting that "bring
your own" config is all-or-nothing scenario. This obviously is not
desired when users want simply to attach their LLMInferenceService to
another gateway.

- Webhook validation now permits such configuration, and only rejects
  when HTTPRoute Spec of LLMInferenceService is also set, treating it as
invalid configuration and suggesting removal of parentRef in favor of
the global Gateway defined as `.spec.router.gateway.refs` instead
- Adjusted EnvTests
- Introduced e2e test to cover this case that leverages llmd-simulator
  for faster feedback loop

Signed-off-by: Bartosz Majsak <[email protected]>

Author: bartoszmajsak

pkg/controller/llmisvc/fixture/gwapi_builders.go

@@ -283,6 +283,12 @@ func WithBackendRefs(refs ...gatewayapi.HTTPBackendRef) HTTPRouteRuleOption {
 	}
 }
 
+func WithHTTPRouteGatewayRef(references ...gatewayapi.ParentReference) HTTPRouteOption {
+	return func(route *gatewayapi.HTTPRoute) {
+		route.Spec.ParentRefs = append(route.Spec.ParentRefs, references...)
+	}
+}
+
 func BackendRefInferencePool(name string) gatewayapi.HTTPBackendRef {
 	return gatewayapi.HTTPBackendRef{
 		BackendRef: gatewayapi.BackendRef{
@@ -334,10 +340,6 @@ func Matches(matches ...gatewayapi.HTTPRouteMatch) HTTPRouteRuleOption {
 	return WithMatches(matches...)
 }
 
-func BackendRefs(refs ...gatewayapi.HTTPBackendRef) HTTPRouteRuleOption {
-	return WithBackendRefs(refs...)
-}
-
 func Timeouts(backendTimeout, requestTimeout string) HTTPRouteRuleOption {
 	return WithTimeouts(backendTimeout, requestTimeout)
 }

pkg/controller/llmisvc/router_gateway_conditions_test.go

@@ -321,7 +321,7 @@ func TestHTTPRouteConditionsEvaluation(t *testing.T) {
 					WithParentRefs(GatewayParentRef("openshift-ai-inference", "openshift-ingress")),
 					WithHTTPRule(
 						Matches(PathPrefixMatch("/llm/facebook-opt-125m-single-simulated")),
-						BackendRefs(ServiceRef("facebook-opt-125m-single-simulated-kserve-workload-svc", 8000, 1)),
+						WithBackendRefs(ServiceRef("facebook-opt-125m-single-simulated-kserve-workload-svc", 8000, 1)),
 						Timeouts("0s", "0s"),
 						Filters(gatewayapi.HTTPRouteFilter{
 							Type: gatewayapi.HTTPRouteFilterURLRewrite,

pkg/controller/llmisvc/validation/llminferenceservice_validator.go

@@ -113,23 +113,6 @@ func (l *LLMInferenceServiceValidator) validateRouterCrossFieldConstraints(llmSv
 	httpRouteRefs := httpRoutePath.Child("refs")
 	httpRouteSpec := httpRoutePath.Child("spec")
 
-	zero := v1alpha1.GatewayRoutesSpec{}
-	if ptr.Deref(router.Route, zero) == zero && router.Gateway != nil && router.Gateway.Refs != nil {
-		return field.ErrorList{
-			field.Invalid(
-				gwRefsPath,
-				router.Gateway.Refs,
-				fmt.Sprintf("unsupported configuration: custom gateway ('%s') cannot be used with managed route ('%s: {}'); "+
-					"either provide your own HTTP routes ('%s') or remove '%s' to use the managed gateway",
-					gwRefsPath,
-					routePath,
-					httpRouteRefs,
-					gwRefsPath,
-				),
-			),
-		}
-	}
-
 	httpRoute := router.Route.HTTP
 	if httpRoute == nil {
 		return field.ErrorList{}
@@ -162,13 +145,14 @@ func (l *LLMInferenceServiceValidator) validateRouterCrossFieldConstraints(llmSv
 		))
 	}
 
-	// Managed route spec cannot be used with user-defined gateway refs
-	if httpRoute.Spec != nil && router.Gateway != nil && len(router.Gateway.Refs) > 0 {
+	// Managed route spec referring to the gateway cannot be used with user-defined gateway refs
+	if httpRoute.Spec != nil && len(httpRoute.Spec.ParentRefs) > 0 &&
+		router.Gateway != nil && len(router.Gateway.Refs) > 0 {
 		allErrs = append(allErrs, field.Invalid(
 			httpRoutePath.Child("spec"),
 			httpRoute.Spec,
 			fmt.Sprintf("unsupported configuration: managed HTTP route spec ('%s') cannot be used with custom gateway refs ('%s'); "+
-				"either remove '%s' or '%s'",
+				"either remove '%s' or '%s.parentRefs'",
 				httpRouteSpec, gwRefsPath, gwRefsPath, httpRouteSpec,
 			),
 		))

pkg/controller/llmisvc/validation/llminferenceservice_validator_int_test.go

@@ -83,7 +83,7 @@ var _ = Describe("LLMInferenceService webhook validation", func() {
 				fixture.WithHTTPRouteSpec(&fixture.HTTPRoute("test-route",
 					fixture.WithHTTPRule(
 						fixture.Matches(fixture.PathPrefixMatch("/test")),
-						fixture.BackendRefs(fixture.ServiceRef("test-service", 80, 1)),
+						fixture.WithBackendRefs(fixture.ServiceRef("test-service", 80, 1)),
 					),
 				).Spec),
 			)
@@ -113,33 +113,17 @@ var _ = Describe("LLMInferenceService webhook validation", func() {
 			Expect(errValidation.Error()).To(ContainSubstring("cannot be used with a managed gateway"))
 		})
 
-		It("should reject LLMInferenceService with managed route and user-defined gateway refs", func(ctx SpecContext) {
-			// given
-			llmSvc := fixture.LLMInferenceService("test-spec-with-gateway-refs",
-				fixture.InNamespace[*v1alpha1.LLMInferenceService](nsName),
-				fixture.WithModelURI("hf://facebook/opt-125m"),
-				fixture.WithGatewayRefs(fixture.LLMGatewayRef("test-gateway", nsName)),
-				fixture.WithManagedRoute(),
-			)
-
-			// when
-			errValidation := envTest.Client.Create(ctx, llmSvc)
-
-			// then
-			Expect(errValidation).To(HaveOccurred())
-			Expect(errValidation.Error()).To(ContainSubstring("cannot be used with managed route"))
-		})
-
-		It("should reject LLMInferenceService with managed route spec and user-defined gateway refs", func(ctx SpecContext) {
+		It("should reject LLMInferenceService with managed route spec with gateway ref and user-defined gateway refs", func(ctx SpecContext) {
 			// given
 			llmSvc := fixture.LLMInferenceService("test-spec-with-gateway-refs",
 				fixture.InNamespace[*v1alpha1.LLMInferenceService](nsName),
 				fixture.WithModelURI("hf://facebook/opt-125m"),
 				fixture.WithGatewayRefs(fixture.LLMGatewayRef("test-gateway", nsName)),
 				fixture.WithHTTPRouteSpec(&fixture.HTTPRoute("test-route",
+					fixture.WithHTTPRouteGatewayRef(fixture.GatewayParentRef("test-gateway", nsName)),
 					fixture.WithHTTPRule(
 						fixture.Matches(fixture.PathPrefixMatch("/test")),
-						fixture.BackendRefs(fixture.ServiceRef("custom-backend", 8080, 1)),
+						fixture.WithBackendRefs(fixture.ServiceRef("custom-backend", 8080, 1)),
 						fixture.Timeouts("30s", "60s"),
 					),
 				).Spec),

test/e2e/llmisvc/README.md

@@ -22,22 +22,32 @@ Tests are marked with both general and cluster-specific capability markers:
 - `@pytest.mark.cluster_nvidia` - NVIDIA GPU tests
 - `@pytest.mark.cluster_nvidia_roce` - NVIDIA ROCe tests
 - `@pytest.mark.cluster_intel` - Intel GPU tests
+- `@pytest.mark.llmd_simulator` - Tests using llm-d simulator instead of real model
 
-Examples:
-```bash
-# Run all LLM inference service tests
+## Examples
+
+### Run all LLM inference service tests 
+
+```shell
 pytest -m "llminferenceservice" test/e2e/llmisvc/
+```
 
-# Run only CPU tests
+### Run only CPU tests
+```shell
 pytest -m "llminferenceservice and cluster_cpu" test/e2e/llmisvc/
+```
 
-# Run only NVIDIA GPU tests
+### Run only NVIDIA GPU tests
+```shell
 pytest -m "llminferenceservice and cluster_nvidia" test/e2e/llmisvc/
+```
 
-# Run all GPU tests (any vendor)
+### Run all GPU tests (any vendor)
+```shell
 pytest -m "llminferenceservice and (cluster_amd or cluster_nvidia or cluster_intel)" test/e2e/llmisvc/
-
-# Run CPU and AMD GPU tests only
+```
+### Run CPU and AMD GPU tests only
+```shell
 pytest -m "llminferenceservice and (cluster_cpu or cluster_amd)" test/e2e/llmisvc/
 ```
 

test/e2e/llmisvc/fixtures.py

@@ -459,7 +459,50 @@
         "router": {
             "scheduler": {},
         },
-    }
+    },
+    "router-with-gateway-ref": {
+        "router": {
+            "gateway": {
+                "refs": [
+                    {"name": "router-gateway-1", "namespace": KSERVE_TEST_NAMESPACE},
+                ],
+            },
+        },
+    },
+    "router-with-managed-route": {
+        "router": {
+            "route": {}
+        },
+    },
+    "workload-llmd-simulator": {
+        "replicas": 1,
+        "model": {"uri": "hf://facebook/opt-125m", "name": "facebook/opt-125m"},
+        "template": {
+            "containers": [
+                {
+                    "name": "main",
+                    "image": "ghcr.io/llm-d/llm-d-inference-sim:v0.5.1",
+                    "command": ["/app/llm-d-inference-sim"],
+                    "args": [
+                        "--port",
+                        "8000",
+                        "--model",
+                        "{{ .Spec.Model.Name }}",
+                        "--mode",
+                        "random",
+                        "--ssl-certfile",
+                        "/etc/ssl/certs/tls.crt",
+                        "--ssl-keyfile",
+                        "/etc/ssl/certs/tls.key"
+                    ],
+                    "resources": {
+                        "limits": {"cpu": "1", "memory": "2Gi"},
+                        "requests": {"cpu": "1", "memory": "2Gi"},
+                    },
+                }
+            ]
+        },
+    },
 }
 
 
@@ -574,7 +617,7 @@ def _get_model_name_from_configs(config_names):
             config = LLMINFERENCESERVICE_CONFIGS[config_name]
             if "model" in config and "name" in config["model"]:
                 return config["model"]["name"]
-    return "default-model"
+    return "default/model"
 
 
 def generate_k8s_safe_suffix(base_name: str, extra_parts: List[str] = None) -> str:
@@ -608,7 +651,7 @@ def generate_test_id(test_case) -> str:
     return "-".join(test_case.base_refs)
 
 
-def create_router_resources(gateways, routes, kserve_client=None):
+def create_router_resources(gateways, routes=None, kserve_client=None):
     if not kserve_client:
         kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
 
@@ -619,7 +662,7 @@ def create_router_resources(gateways, routes, kserve_client=None):
         for gateway in gateways:
             create_or_update_gateway(kserve_client, gateway)
             gateways_created.append(gateway)
-        for route in routes:
+        for route in routes or []:
             create_or_update_route(kserve_client, route)
             routes_created.append(route)
     except Exception as e:
@@ -628,11 +671,11 @@ def create_router_resources(gateways, routes, kserve_client=None):
         raise
 
 
-def delete_router_resources(gateways, routes, kserve_client=None):
+def delete_router_resources(gateways, routes=None, kserve_client=None):
     if not kserve_client:
         kserve_client = KServeClient(config_file=os.environ.get("KUBECONFIG", "~/.kube/config"))
 
-    for route in routes:
+    for route in routes or []:
         try:
             logger.info(f"Cleaning up HttpRoute {route.get('metadata', {}).get('name')}")
             delete_route(kserve_client, route.get("metadata", {}).get("name"), route.get("metadata", {}).get("namespace", "default"))

test/e2e/llmisvc/test_llm_inference_service.py

@@ -83,6 +83,24 @@ class TestCase:
 @pytest.mark.parametrize(
     "test_case",
     [
+        pytest.param(
+            TestCase(
+                base_refs=[
+                    "router-with-gateway-ref",
+                    "router-with-managed-route",
+                    "model-fb-opt-125m",
+                    "workload-llmd-simulator",
+                ],
+                prompt="KServe is a",
+                before_test=[lambda: create_router_resources(
+                    gateways=[ROUTER_GATEWAYS[0]],
+                )],
+                after_test=[lambda: delete_router_resources(
+                    gateways=[ROUTER_GATEWAYS[0]],
+                )],
+            ),
+            marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_single_node, pytest.mark.llmd_simulator],
+        ),
         pytest.param(
             TestCase(
                 base_refs=[

test/e2e/pytest.ini

@@ -23,4 +23,6 @@ markers =
     cluster_amd: test targeting cluster with AMD
     cluster_intel: test targeting cluster with Intel
     cluster_nvidia: test targeting cluster with NVIDIA
-    cluster_single_node: test targeting single node cluster
+    cluster_nvidia_roce: test targeting cluster with NVIDIA ROCe
+    cluster_single_node: test targeting single node cluster
+    llmd_simulator: test using llm-d simulator