From d3786ecdff9fa289e17d38749aabb335a0006af0 Mon Sep 17 00:00:00 2001 From: William Zhang Date: Wed, 3 Dec 2025 01:48:17 -0800 Subject: [PATCH 1/3] add tls and filters to ir for custom backend ref Signed-off-by: William Zhang --- internal/gatewayapi/route.go | 18 +- .../httproute-with-custom-backend-tls.in.yaml | 65 +++++ ...httproute-with-custom-backend-tls.out.yaml | 222 ++++++++++++++++++ internal/xds/translator/cluster.go | 2 +- internal/xds/translator/route.go | 2 +- .../http-route-custom-backend-tls.yaml | 39 +++ ...ttp-route-custom-backend-tls.clusters.yaml | 46 ++++ ...tp-route-custom-backend-tls.endpoints.yaml | 5 + ...tp-route-custom-backend-tls.listeners.yaml | 35 +++ .../http-route-custom-backend-tls.routes.yaml | 32 +++ ...http-route-custom-backend-tls.secrets.yaml | 4 + internal/xds/translator/translator_test.go | 1 + 12 files changed, 463 insertions(+), 8 deletions(-) create mode 100644 internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.in.yaml create mode 100644 internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.out.yaml create mode 100644 internal/xds/translator/testdata/in/extension-xds-ir/http-route-custom-backend-tls.yaml create mode 100644 internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.routes.yaml create mode 100644 internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.secrets.yaml diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index f9ce28d2eb5..581fa0a15f7 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -1780,6 +1780,7 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe return emptyDS, nil, status.NewRouteStatusError(tlsErr, status.RouteReasonInvalidBackendTLS) } + isCustomBackend := false switch KindDerefOr(backendRef.Kind, resource.KindService) { case resource.KindServiceImport: ds, err = t.processServiceImportDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol, envoyProxy) @@ -1798,7 +1799,8 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe ds = t.processBackendDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol) default: // Handle custom backend resources defined in extension manager - if t.isCustomBackendResource(backendRef.Group, KindDerefOr(backendRef.Kind, resource.KindService)) { + isCustomBackend = t.isCustomBackendResource(backendRef.Group, KindDerefOr(backendRef.Kind, resource.KindService)) + if isCustomBackend { // Add the custom backend resource to ExtensionRefFilters so it can be processed by the extension system unstructuredRef = t.processBackendExtensions(backendRef.BackendObjectReference, backendNamespace, resources) @@ -1813,11 +1815,10 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe ).WithType(gwapiv1.RouteConditionResolvedRefs) } - return &ir.DestinationSetting{ + ds = &ir.DestinationSetting{ Name: name, - Weight: &weight, IsCustomBackend: true, - }, unstructuredRef, nil + } } } @@ -1829,11 +1830,16 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe return emptyDS, nil, status.NewRouteStatusError(filtersErr, status.RouteReasonInvalidBackendFilters) } + ds.Weight = &weight + + if isCustomBackend { + return ds, unstructuredRef, nil + } + if err := validateDestinationSettings(ds, t.IsEnvoyServiceRouting(envoyProxy), backendRef.Kind); err != nil { return emptyDS, nil, err } - ds.Weight = &weight return ds, nil, nil } @@ -2334,7 +2340,7 @@ func (t *Translator) getTargetBackendReference( } default: - // Set the section name to the port number if the backend is a EG Backend + // Set the section name to the port number if the backend is a EG Backend or a custom backend ref.SectionName = SectionNamePtr(strconv.Itoa(int(*backendRef.Port))) } diff --git a/internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.in.yaml b/internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.in.yaml new file mode 100644 index 00000000000..bfc9dc37875 --- /dev/null +++ b/internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.in.yaml @@ -0,0 +1,65 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + hostname: "*.envoyproxy.io" + allowedRoutes: + namespaces: + from: All +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/s3" + backendRefs: + - group: storage.example.io + kind: S3Backend + name: s3-backend + port: 443 +backendTLSPolicies: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: BackendTLSPolicy + metadata: + name: policy-btls-for-custom-backend + namespace: default + spec: + targetRefs: + - group: storage.example.io + kind: S3Backend + name: s3-backend + validation: + wellKnownCACertificates: System + hostname: example.com + subjectAltNames: + - type: Hostname + hostname: s3.amazonaws.com +extensionRefFilters: +- apiVersion: storage.example.io/v1alpha1 + kind: S3Backend + metadata: + name: s3-backend + namespace: default + spec: + bucket: my-s3-bucket + region: us-west-2 + endpoint: s3.amazonaws.com diff --git a/internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.out.yaml b/internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.out.yaml new file mode 100644 index 00000000000..da3959568f4 --- /dev/null +++ b/internal/gatewayapi/testdata/extensions/httproute-with-custom-backend-tls.out.yaml @@ -0,0 +1,222 @@ +backendTLSPolicies: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: BackendTLSPolicy + metadata: + name: policy-btls-for-custom-backend + namespace: default + spec: + targetRefs: + - group: storage.example.io + kind: S3Backend + name: s3-backend + validation: + hostname: example.com + subjectAltNames: + - hostname: s3.amazonaws.com + type: Hostname + wellKnownCACertificates: System + status: + ancestors: + - ancestorRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http + conditions: + - lastTransitionTime: null + message: Resolved all the Object references. + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + - lastTransitionTime: null + message: Policy has been accepted. + reason: Accepted + status: "True" + type: Accepted + controllerName: gateway.envoyproxy.io/gatewayclass-controller +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + hostname: '*.envoyproxy.io' + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - group: storage.example.io + kind: S3Backend + name: s3-backend + port: 443 + matches: + - path: + value: /s3 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + ownerReference: + kind: GatewayClass + name: envoy-gateway-class + name: envoy-gateway/gateway-1 + namespace: "" +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + globalResources: + proxyServiceCluster: + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + sectionName: "8080" + name: envoy-gateway/gateway-1 + settings: + - addressType: IP + endpoints: + - host: 7.6.5.4 + port: 8080 + zone: zone1 + metadata: + kind: Service + name: envoy-envoy-gateway-gateway-1-196ae069 + sectionName: "8080" + name: envoy-gateway/gateway-1 + protocol: TCP + http: + - address: 0.0.0.0 + externalPort: 80 + hostnames: + - '*.envoyproxy.io' + isHTTP2: false + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0 + settings: + - isCustomBackend: true + name: httproute/default/httproute-1/rule/0/backend/0 + tls: + alpnProtocols: null + caCertificate: + name: policy-btls-for-custom-backend/default-ca + sni: example.com + subjectAltNames: + - hostname: s3.amazonaws.com + useSystemTrustStore: true + weight: 1 + extensionRefs: + - object: + apiVersion: storage.example.io/v1alpha1 + kind: S3Backend + metadata: + name: s3-backend + namespace: default + spec: + bucket: my-s3-bucket + endpoint: s3.amazonaws.com + region: us-west-2 + hostname: gateway.envoyproxy.io + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: /s3 + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index f35c85be7ef..cbc27647925 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -249,7 +249,7 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { matchName := fmt.Sprintf("%s/tls/%d", args.name, i) // Dynamic resolver clusters have no endpoints, so we need to set the transport socket directly. - if args.endpointType == EndpointTypeDynamicResolver { + if args.endpointType == EndpointTypeDynamicResolver || ds.IsCustomBackend { cluster.TransportSocket = socket } else { cluster.TransportSocketMatches = append(cluster.TransportSocketMatches, &clusterv3.Cluster_TransportSocketMatch{ diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index b60e3db5a61..15794b1c5cc 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -326,7 +326,7 @@ func buildXdsWeightedRouteAction(backendWeights *ir.BackendWeights, settings []* } for _, destinationSetting := range settings { - if len(destinationSetting.Endpoints) > 0 || destinationSetting.IsDynamicResolver { // Dynamic resolver has no endpoints + if len(destinationSetting.Endpoints) > 0 || destinationSetting.IsDynamicResolver || destinationSetting.IsCustomBackend { // Dynamic resolver and custom backends have no endpoints validCluster := &routev3.WeightedCluster_ClusterWeight{ Name: destinationSetting.Name, Weight: &wrapperspb.UInt32Value{Value: *destinationSetting.Weight}, diff --git a/internal/xds/translator/testdata/in/extension-xds-ir/http-route-custom-backend-tls.yaml b/internal/xds/translator/testdata/in/extension-xds-ir/http-route-custom-backend-tls.yaml new file mode 100644 index 00000000000..a2cdd43b929 --- /dev/null +++ b/internal/xds/translator/testdata/in/extension-xds-ir/http-route-custom-backend-tls.yaml @@ -0,0 +1,39 @@ +http: +- name: "custom-backend-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "custom-backend-route" + hostname: "*" + pathMatch: + prefix: "/" + destination: + name: "custom-backend-dest" + settings: + - isCustomBackend: true + name: "custom-backend-dest/backend/0" + tls: + caCertificate: + name: "policy-btls-for-inference-pool/default-ca" + sni: "inference.example.com" + subjectAltNames: + - hostname: "inference-pool.example.com" + useSystemTrustStore: true + weight: 1 + extensionRefs: + - object: + apiVersion: inference.networking.x-k8s.io/v1alpha2 + kind: InferencePool + metadata: + name: inference-pool + spec: + targetPortNumber: 8000 + selector: + app: vllm-llama3-8b-instruct + extensionRef: + name: vllm-llama3-8b-instruct-epp diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.clusters.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.clusters.yaml new file mode 100644 index 00000000000..3ecd9793572 --- /dev/null +++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.clusters.yaml @@ -0,0 +1,46 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + ignoreHealthOnHostRemoval: true + lbPolicy: CLUSTER_PROVIDED + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: custom-backend-dest + originalDstLbConfig: + httpHeaderName: x-gateway-destination-endpoint + useHttpHeader: true + perConnectionBufferLimitBytes: 32768 + transportSocket: + name: envoy.transport_sockets.tls + typedConfig: + '@type': type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + commonTlsContext: + combinedValidationContext: + defaultValidationContext: + matchTypedSubjectAltNames: + - matcher: + exact: inference-pool.example.com + sanType: DNS + validationContextSdsSecretConfig: + name: policy-btls-for-inference-pool/default-ca + sdsConfig: + ads: {} + resourceApiVersion: V3 + sni: inference.example.com + type: ORIGINAL_DST + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + autoConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + httpProtocolOptions: {} diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.endpoints.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.endpoints.yaml new file mode 100644 index 00000000000..11ec2182a6d --- /dev/null +++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.endpoints.yaml @@ -0,0 +1,5 @@ +- clusterName: custom-backend-dest + endpoints: + - loadBalancingWeight: 1 + locality: + region: custom-backend-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.listeners.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.listeners.yaml new file mode 100644 index 00000000000..67ce52cb2bc --- /dev/null +++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.listeners.yaml @@ -0,0 +1,35 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: custom-backend-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: custom-backend-listener + maxConnectionsToAcceptPerSocketEvent: 1 + name: custom-backend-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.routes.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.routes.yaml new file mode 100644 index 00000000000..5fd4a3a946a --- /dev/null +++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.routes.yaml @@ -0,0 +1,32 @@ +- ignorePortInHostMatching: true + name: custom-backend-listener + virtualHosts: + - domains: + - '*' + name: custom-backend-listener/* + routes: + - match: + prefix: / + name: custom-backend-route + responseHeadersToAdd: + - header: + key: mock-extension-was-here-route-name + value: custom-backend-route + - header: + key: mock-extension-was-here-route-hostnames + value: '*' + - header: + key: mock-extension-was-here-extensionRef-name + value: inference-pool + - header: + key: mock-extension-was-here-extensionRef-namespace + - header: + key: mock-extension-was-here-extensionRef-kind + value: InferencePool + - header: + key: mock-extension-was-here-extensionRef-apiversion + value: inference.networking.x-k8s.io/v1alpha2 + route: + cluster: custom-backend-dest + upgradeConfigs: + - upgradeType: websocket diff --git a/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.secrets.yaml b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.secrets.yaml new file mode 100644 index 00000000000..841028d7d44 --- /dev/null +++ b/internal/xds/translator/testdata/out/extension-xds-ir/http-route-custom-backend-tls.secrets.yaml @@ -0,0 +1,4 @@ +- name: policy-btls-for-inference-pool/default-ca + validationContext: + trustedCa: + filename: /etc/ssl/certs/ca-certificates.crt diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index 0f1849377f7..8ccac5c9342 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -298,6 +298,7 @@ func TestTranslateXdsWithExtensionErrorsWhenFailOpen(t *testing.T) { "http-route-extension-translate-error": {}, "multiple-listeners-same-port-error": {}, "http-route-custom-backend": {}, + "http-route-custom-backend-tls": {}, "http-route-custom-backends-multiple": {}, "http-route-custom-backends-partial": {}, "http-route-custom-backend-error": {}, From bc645adccc1be23d9e1ae0fc1123df8b2f0bbae1 Mon Sep 17 00:00:00 2001 From: William Zhang Date: Wed, 3 Dec 2025 21:00:25 -0800 Subject: [PATCH 2/3] add comment clarifying transport socket for custom backend Signed-off-by: William Zhang --- internal/xds/translator/cluster.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index cbc27647925..171f7c10db5 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -249,6 +249,8 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { matchName := fmt.Sprintf("%s/tls/%d", args.name, i) // Dynamic resolver clusters have no endpoints, so we need to set the transport socket directly. + // Since not much is known about custom backends, we just provide a top-level transport socket directly. + // Extension servers can then either use or reference this transport socket to build their own configurations. if args.endpointType == EndpointTypeDynamicResolver || ds.IsCustomBackend { cluster.TransportSocket = socket } else { From 3e44ea378ef3f4f8287afc694d408e7bec99911b Mon Sep 17 00:00:00 2001 From: William Zhang Date: Wed, 3 Dec 2025 21:08:32 -0800 Subject: [PATCH 3/3] add release note Signed-off-by: William Zhang --- release-notes/current.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/release-notes/current.yaml b/release-notes/current.yaml index 5d3192e2585..9d16e7576a0 100644 --- a/release-notes/current.yaml +++ b/release-notes/current.yaml @@ -11,6 +11,7 @@ security updates: | # New features or capabilities added in this release. new features: | Added support for weight in BackendRef API to enable traffic splitting for non-x-route resources. + Added support for targeting custom backend refs managed by extension servers with configurations such as BackendTLSPolicy. bug fixes: | Fixed configured OIDC authorization endpoint being overridden by discovered endpoints from issuer's well-known URL.