feat: support adding tls and filter configs to the IRs built for a custom backend ref

internal/gatewayapi/route.go

@@ -1780,6 +1780,7 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe
 		return emptyDS, nil, status.NewRouteStatusError(tlsErr, status.RouteReasonInvalidBackendTLS)
 	}
 
+	isCustomBackend := false
 	switch KindDerefOr(backendRef.Kind, resource.KindService) {
 	case resource.KindServiceImport:
 		ds, err = t.processServiceImportDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol, envoyProxy)
@@ -1798,7 +1799,8 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe
 		ds = t.processBackendDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol)
 	default:
 		// Handle custom backend resources defined in extension manager
-		if t.isCustomBackendResource(backendRef.Group, KindDerefOr(backendRef.Kind, resource.KindService)) {
+		isCustomBackend = t.isCustomBackendResource(backendRef.Group, KindDerefOr(backendRef.Kind, resource.KindService))
+		if isCustomBackend {
 			// Add the custom backend resource to ExtensionRefFilters so it can be processed by the extension system
 			unstructuredRef = t.processBackendExtensions(backendRef.BackendObjectReference, backendNamespace, resources)
 
@@ -1813,11 +1815,10 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe
 				).WithType(gwapiv1.RouteConditionResolvedRefs)
 			}
 
-			return &ir.DestinationSetting{
+			ds = &ir.DestinationSetting{
 				Name:            name,
-				Weight:          &weight,
 				IsCustomBackend: true,
-			}, unstructuredRef, nil
+			}
 		}
 	}
 
@@ -1829,11 +1830,16 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe
 		return emptyDS, nil, status.NewRouteStatusError(filtersErr, status.RouteReasonInvalidBackendFilters)
 	}
 
+	ds.Weight = &weight
+
+	if isCustomBackend {
+		return ds, unstructuredRef, nil
+	}
+
 	if err := validateDestinationSettings(ds, t.IsEnvoyServiceRouting(envoyProxy), backendRef.Kind); err != nil {
 		return emptyDS, nil, err
 	}
 
-	ds.Weight = &weight
 	return ds, nil, nil
 }
 
@@ -2334,7 +2340,7 @@ func (t *Translator) getTargetBackendReference(
 		}
 
 	default:
-		// Set the section name to the port number if the backend is a EG Backend
+		// Set the section name to the port number if the backend is a EG Backend or a custom backend
 		ref.SectionName = SectionNamePtr(strconv.Itoa(int(*backendRef.Port)))
 	}
 

internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.in.yaml

@@ -0,0 +1,65 @@
+gateways:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: Gateway
+  metadata:
+    namespace: envoy-gateway
+    name: gateway-1
+  spec:
+    gatewayClassName: envoy-gateway-class
+    listeners:
+    - name: http
+      protocol: HTTP
+      port: 80
+      hostname: "*.envoyproxy.io"
+      allowedRoutes:
+        namespaces:
+          from: All
+httpRoutes:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: HTTPRoute
+  metadata:
+    namespace: default
+    name: httproute-1
+  spec:
+    hostnames:
+    - gateway.envoyproxy.io
+    parentRefs:
+    - namespace: envoy-gateway
+      name: gateway-1
+      sectionName: http
+    rules:
+    - matches:
+      - path:
+          value: "/s3"
+      backendRefs:
+      - group: storage.example.io
+        kind: S3Backend
+        name: s3-backend
+        port: 443
+backendTLSPolicies:
+- apiVersion: gateway.networking.k8s.io/v1alpha2
+  kind: BackendTLSPolicy
+  metadata:
+    name: policy-btls-for-custom-backend
+    namespace: default
+  spec:
+    targetRefs:
+    - group: storage.example.io
+      kind: S3Backend
+      name: s3-backend
+    validation:
+      wellKnownCACertificates: System
+      hostname: example.com
+      subjectAltNames:
+      - type: Hostname
+        hostname: s3.amazonaws.com
+extensionRefFilters:
+- apiVersion: storage.example.io/v1alpha1
+  kind: S3Backend
+  metadata:
+    name: s3-backend
+    namespace: default
+  spec:
+    bucket: my-s3-bucket
+    region: us-west-2
+    endpoint: s3.amazonaws.com

internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.out.yaml

@@ -0,0 +1,222 @@
+backendTLSPolicies:
+- apiVersion: gateway.networking.k8s.io/v1alpha2
+  kind: BackendTLSPolicy
+  metadata:
+    name: policy-btls-for-custom-backend
+    namespace: default
+  spec:
+    targetRefs:
+    - group: storage.example.io
+      kind: S3Backend
+      name: s3-backend
+    validation:
+      hostname: example.com
+      subjectAltNames:
+      - hostname: s3.amazonaws.com
+        type: Hostname
+      wellKnownCACertificates: System
+  status:
+    ancestors:
+    - ancestorRef:
+        name: gateway-1
+        namespace: envoy-gateway
+        sectionName: http
+      conditions:
+      - lastTransitionTime: null
+        message: Resolved all the Object references.
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      - lastTransitionTime: null
+        message: Policy has been accepted.
+        reason: Accepted
+        status: "True"
+        type: Accepted
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+gateways:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: Gateway
+  metadata:
+    name: gateway-1
+    namespace: envoy-gateway
+  spec:
+    gatewayClassName: envoy-gateway-class
+    listeners:
+    - allowedRoutes:
+        namespaces:
+          from: All
+      hostname: '*.envoyproxy.io'
+      name: http
+      port: 80
+      protocol: HTTP
+  status:
+    listeners:
+    - attachedRoutes: 1
+      conditions:
+      - lastTransitionTime: null
+        message: Sending translated listener configuration to the data plane
+        reason: Programmed
+        status: "True"
+        type: Programmed
+      - lastTransitionTime: null
+        message: Listener has been successfully translated
+        reason: Accepted
+        status: "True"
+        type: Accepted
+      - lastTransitionTime: null
+        message: Listener references have been resolved
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      name: http
+      supportedKinds:
+      - group: gateway.networking.k8s.io
+        kind: HTTPRoute
+      - group: gateway.networking.k8s.io
+        kind: GRPCRoute
+httpRoutes:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: HTTPRoute
+  metadata:
+    name: httproute-1
+    namespace: default
+  spec:
+    hostnames:
+    - gateway.envoyproxy.io
+    parentRefs:
+    - name: gateway-1
+      namespace: envoy-gateway
+      sectionName: http
+    rules:
+    - backendRefs:
+      - group: storage.example.io
+        kind: S3Backend
+        name: s3-backend
+        port: 443
+      matches:
+      - path:
+          value: /s3
+  status:
+    parents:
+    - conditions:
+      - lastTransitionTime: null
+        message: Route is accepted
+        reason: Accepted
+        status: "True"
+        type: Accepted
+      - lastTransitionTime: null
+        message: Resolved all the Object references for the Route
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+      parentRef:
+        name: gateway-1
+        namespace: envoy-gateway
+        sectionName: http
+infraIR:
+  envoy-gateway/gateway-1:
+    proxy:
+      listeners:
+      - address: null
+        name: envoy-gateway/gateway-1/http
+        ports:
+        - containerPort: 10080
+          name: http-80
+          protocol: HTTP
+          servicePort: 80
+      metadata:
+        labels:
+          gateway.envoyproxy.io/owning-gateway-name: gateway-1
+          gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway
+        ownerReference:
+          kind: GatewayClass
+          name: envoy-gateway-class
+      name: envoy-gateway/gateway-1
+      namespace: ""
+xdsIR:
+  envoy-gateway/gateway-1:
+    accessLog:
+      json:
+      - path: /dev/stdout
+    globalResources:
+      proxyServiceCluster:
+        metadata:
+          kind: Service
+          name: envoy-envoy-gateway-gateway-1-196ae069
+          sectionName: "8080"
+        name: envoy-gateway/gateway-1
+        settings:
+        - addressType: IP
+          endpoints:
+          - host: 7.6.5.4
+            port: 8080
+            zone: zone1
+          metadata:
+            kind: Service
+            name: envoy-envoy-gateway-gateway-1-196ae069
+            sectionName: "8080"
+          name: envoy-gateway/gateway-1
+          protocol: TCP
+    http:
+    - address: 0.0.0.0
+      externalPort: 80
+      hostnames:
+      - '*.envoyproxy.io'
+      isHTTP2: false
+      metadata:
+        kind: Gateway
+        name: gateway-1
+        namespace: envoy-gateway
+        sectionName: http
+      name: envoy-gateway/gateway-1/http
+      path:
+        escapedSlashesAction: UnescapeAndRedirect
+        mergeSlashes: true
+      port: 10080
+      routes:
+      - destination:
+          metadata:
+            kind: HTTPRoute
+            name: httproute-1
+            namespace: default
+          name: httproute/default/httproute-1/rule/0
+          settings:
+          - isCustomBackend: true
+            name: httproute/default/httproute-1/rule/0/backend/0
+            tls:
+              alpnProtocols: null
+              caCertificate:
+                name: policy-btls-for-custom-backend/default-ca
+              sni: example.com
+              subjectAltNames:
+              - hostname: s3.amazonaws.com
+              useSystemTrustStore: true
+            weight: 1
+        extensionRefs:
+        - object:
+            apiVersion: storage.example.io/v1alpha1
+            kind: S3Backend
+            metadata:
+              name: s3-backend
+              namespace: default
+            spec:
+              bucket: my-s3-bucket
+              endpoint: s3.amazonaws.com
+              region: us-west-2
+        hostname: gateway.envoyproxy.io
+        isHTTP2: false
+        metadata:
+          kind: HTTPRoute
+          name: httproute-1
+          namespace: default
+        name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io
+        pathMatch:
+          distinct: false
+          name: ""
+          prefix: /s3
+    readyListener:
+      address: 0.0.0.0
+      ipFamily: IPv4
+      path: /ready
+      port: 19003

internal/xds/translator/cluster.go

@@ -249,7 +249,9 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) {
 			matchName := fmt.Sprintf("%s/tls/%d", args.name, i)
 
 			// Dynamic resolver clusters have no endpoints, so we need to set the transport socket directly.
-			if args.endpointType == EndpointTypeDynamicResolver {
+			// Since not much is known about custom backends, we just provide a top-level transport socket directly.
+			// Extension servers can then either use or reference this transport socket to build their own configurations.
+			if args.endpointType == EndpointTypeDynamicResolver || ds.IsCustomBackend {
 				cluster.TransportSocket = socket
 			} else {
 				cluster.TransportSocketMatches = append(cluster.TransportSocketMatches, &clusterv3.Cluster_TransportSocketMatch{

internal/xds/translator/route.go

@@ -326,7 +326,7 @@ func buildXdsWeightedRouteAction(backendWeights *ir.BackendWeights, settings []*
 	}
 
 	for _, destinationSetting := range settings {
-		if len(destinationSetting.Endpoints) > 0 || destinationSetting.IsDynamicResolver { // Dynamic resolver has no endpoints
+		if len(destinationSetting.Endpoints) > 0 || destinationSetting.IsDynamicResolver || destinationSetting.IsCustomBackend { // Dynamic resolver and custom backends have no endpoints
 			validCluster := &routev3.WeightedCluster_ClusterWeight{
 				Name:   destinationSetting.Name,
 				Weight: &wrapperspb.UInt32Value{Value: *destinationSetting.Weight},