controller: isolate invalid backends in config propagation (#1174)

**Description**

This changes the logic in the config construction in Gateway object
controller where we collect all the backends as well as the attached BSP
in a way that invalid backend/bsp won't block others to be reconciled.

**Related Issues/PRs (if applicable)**

Fixes #1122

Signed-off-by: Takeshi Yoneda <[email protected]>

Author: mathetake

internal/controller/gateway.go

@@ -166,8 +166,8 @@ func (c *GatewayController) reconcileFilterConfigSecret(ctx context.Context, con
 	for i := range aiGatewayRoutes {
 		aiGatewayRoute := &aiGatewayRoutes[i]
 		spec := aiGatewayRoute.Spec
-		for i := range spec.Rules {
-			rule := &spec.Rules[i]
+		for ruleIndex := range spec.Rules {
+			rule := &spec.Rules[ruleIndex]
 			for _, m := range rule.Matches {
 				for _, h := range m.Headers {
 					// If explicitly set to something that is not an exact match, skip.
@@ -184,10 +184,10 @@ func (c *GatewayController) reconcileFilterConfigSecret(ctx context.Context, con
 					})
 				}
 			}
-			for j := range rule.BackendRefs {
-				backendRef := &rule.BackendRefs[j]
+			for backendRefIndex := range rule.BackendRefs {
+				backendRef := &rule.BackendRefs[backendRefIndex]
 				b := filterapi.Backend{}
-				b.Name = internalapi.PerRouteRuleRefBackendName(aiGatewayRoute.Namespace, backendRef.Name, aiGatewayRoute.Name, i, j)
+				b.Name = internalapi.PerRouteRuleRefBackendName(aiGatewayRoute.Namespace, backendRef.Name, aiGatewayRoute.Name, ruleIndex, backendRefIndex)
 				b.ModelNameOverride = backendRef.ModelNameOverride
 				if backendRef.IsInferencePool() {
 					// We assume that InferencePools are all OpenAI schema.
@@ -197,14 +197,20 @@ func (c *GatewayController) reconcileFilterConfigSecret(ctx context.Context, con
 					var bsp *aigv1a1.BackendSecurityPolicy
 					backendObj, bsp, err = c.backendWithMaybeBSP(ctx, aiGatewayRoute.Namespace, backendRef.Name)
 					if err != nil {
-						return fmt.Errorf("failed to get AIServiceBackend %s: %w", b.Name, err)
+						c.logger.Error(err, "failed to get backend or backend security policy. Skipping this backend.",
+							"backend_name", backendRef.Name, "aigatewayroute", aiGatewayRoute.Name,
+							"namespace", aiGatewayRoute.Namespace)
+						continue
 					}
 					b.HeaderMutation = headerMutationToFilterAPI(backendObj.Spec.HeaderMutation)
 					b.Schema = schemaToFilterAPI(backendObj.Spec.APISchema)
 					if bsp != nil {
 						b.Auth, err = c.bspToFilterAPIBackendAuth(ctx, bsp)
 						if err != nil {
-							return fmt.Errorf("failed to create backend auth: %w", err)
+							c.logger.Error(err, "failed to get backend auth from backend security policy. Skipping this backend.",
+								"backend_name", backendRef.Name, "backend_security_policy", bsp.Name,
+								"aigatewayroute", aiGatewayRoute.Name, "namespace", aiGatewayRoute.Namespace)
+							continue
 						}
 					}
 				}

internal/controller/gateway_test.go

@@ -163,7 +163,6 @@ func TestGatewayController_reconcileFilterConfigSecret(t *testing.T) {
 	kube := fake2.NewClientset()
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zap.Options{Development: true, Level: zapcore.DebugLevel})))
 	c := NewGatewayController(fakeClient, kube, ctrl.Log,
-
 		"docker.io/envoyproxy/ai-gateway-extproc:latest", false, nil)
 
 	const gwNamespace = "ns"
@@ -173,7 +172,11 @@ func TestGatewayController_reconcileFilterConfigSecret(t *testing.T) {
 			Spec: aigv1a1.AIGatewayRouteSpec{
 				Rules: []aigv1a1.AIGatewayRouteRule{
 					{
-						BackendRefs: []aigv1a1.AIGatewayRouteRuleBackendRef{{Name: "apple"}},
+						BackendRefs: []aigv1a1.AIGatewayRouteRuleBackendRef{
+							{Name: "apple"},
+							{Name: "invalid-bsp-backend"},  // This should be ignored as the BSP is invalid.
+							{Name: "non-existent-backend"}, // This should be ignored as the backend does not exist.
+						},
 						Matches: []aigv1a1.AIGatewayRouteRuleMatch{
 							{
 								Headers: []gwapiv1.HTTPHeaderMatch{
@@ -221,11 +224,36 @@ func TestGatewayController_reconcileFilterConfigSecret(t *testing.T) {
 				BackendRef: gwapiv1.BackendObjectReference{Name: "some-backend1", Namespace: ptr.To[gwapiv1.Namespace](gwNamespace)},
 			},
 		},
+		{
+			ObjectMeta: metav1.ObjectMeta{Name: "invalid-bsp-backend", Namespace: gwNamespace},
+			Spec: aigv1a1.AIServiceBackendSpec{
+				BackendRef: gwapiv1.BackendObjectReference{Name: "some-backend1", Namespace: ptr.To[gwapiv1.Namespace](gwNamespace)},
+			},
+		},
 	} {
 		err := fakeClient.Create(t.Context(), aigwRoute)
 		require.NoError(t, err)
 	}
 
+	// Create a BackendSecurityPolicy that is invalid (missing secret ref).
+	err := fakeClient.Create(t.Context(), &aigv1a1.BackendSecurityPolicy{
+		ObjectMeta: metav1.ObjectMeta{Name: "invalid-bsp", Namespace: gwNamespace},
+		Spec: aigv1a1.BackendSecurityPolicySpec{
+			Type: aigv1a1.BackendSecurityPolicyTypeAPIKey,
+			APIKey: &aigv1a1.BackendSecurityPolicyAPIKey{
+				SecretRef: &gwapiv1.SecretObjectReference{Name: "non-existent-secret"},
+			},
+			TargetRefs: []gwapiv1a2.LocalPolicyTargetReference{
+				{
+					Kind:  "AIServiceBackend",
+					Group: "aigateway.envoyproxy.io",
+					Name:  "invalid-bsp-backend",
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+
 	for range 2 { // Reconcile twice to make sure the secret update path is working.
 		const someNamespace = "some-namespace"
 		configName := FilterConfigSecretPerGatewayName("gw", gwNamespace)