Issue 3138 - Conformance Tests for BackendTLSPolicy - normative (#3212)

* Issue 3138 - add normative conformance test for

BackendTLSPolicy.

* Fix where the new go module needed by echo-basic resides, but keep the original as well.
See https://github.com/kubernetes-sigs/gateway-api/pull/2745/files#diff-cf5b4a9b433acc91f8c7cc2dc802e29aa712c8d820887742f3bc5ae45b7a9d0fR24-R28

* Rebase and make requested updates.

* Update tests after implementation testing
conformance/base/manifests.yaml - fix yaml
conformance/tests/backendtlspolicy.yaml - fix yaml
conformance/tests/tlsroute-simple-same-namespace.go - rename cert for sharing
conformance/utils/suite/conformance.go - fix a bug in cleanup-base-resources flag application
conformance/utils/suite/suite.go - rename cert for sharing

* Incorporate Shane and Flynn's feedback

* Add unit testing for generateCACert, new HTTPS
call, some debugging, and fix yaml

* Update echo-basic images

* Fix lint errors and condition evaluation in tests

* Fix yaml for httpRoute and backendTLSPolicy.  Fix CA generation.
Fix certificate unit test.

* Refactor test, fix yaml

* Fix the tests for normative BackendTLSPolicy

# Conflicts:
#	conformance/utils/http/http.go

* Make changes from review comments.
Add conformance profiles to logged information.

* Address further review comments

* Address review comments:

Remove echo-basic changes, fix cert building, and adjust the port used for gateways with multiple listeners

Co-authored-by: Norwin Schnyder <[email protected]>

* Address the last of the review comments

---------

Co-authored-by: Norwin Schnyder <[email protected]>

Author: candita

conformance/base/manifests.yaml

@@ -138,7 +138,7 @@ spec:
     spec:
       containers:
       - name: infra-backend-v1
-        # From https://github.com/kubernetes-sigs/ingress-controller-conformance/tree/master/images/echoserver
+        # Originally from https://github.com/kubernetes-sigs/ingress-controller-conformance/tree/master/images/echoserver
         image: gcr.io/k8s-staging-gateway-api/echo-basic:v20240412-v1.0.0-394-g40c666fd
         env:
         - name: POD_NAME
@@ -300,7 +300,7 @@ spec:
       volumes:
       - name: secret-volume
         secret:
-          secretName: tls-passthrough-checks-certificate
+          secretName: tls-checks-certificate
           items:
           - key: tls.crt
             path: crt

conformance/conformance.go

@@ -29,14 +29,11 @@ import (
 	"sigs.k8s.io/gateway-api/conformance/tests"
 	conformanceconfig "sigs.k8s.io/gateway-api/conformance/utils/config"
 	"sigs.k8s.io/gateway-api/conformance/utils/flags"
-
 	"sigs.k8s.io/gateway-api/conformance/utils/suite"
 
 	"github.com/stretchr/testify/require"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-
 	clientset "k8s.io/client-go/kubernetes"
-
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/yaml"
@@ -66,7 +63,6 @@ func DefaultOptions(t *testing.T) suite.ConformanceOptions {
 
 	supportedFeatures := suite.ParseSupportedFeatures(*flags.SupportedFeatures)
 	exemptFeatures := suite.ParseSupportedFeatures(*flags.ExemptFeatures)
-
 	skipTests := suite.ParseSkipTests(*flags.SkipTests)
 	namespaceLabels := suite.ParseKeyValuePairs(*flags.NamespaceLabels)
 	namespaceAnnotations := suite.ParseKeyValuePairs(*flags.NamespaceAnnotations)

conformance/tests/backendtlspolicy.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tests
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/types"
+
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	h "sigs.k8s.io/gateway-api/conformance/utils/http"
+	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
+	"sigs.k8s.io/gateway-api/conformance/utils/suite"
+	"sigs.k8s.io/gateway-api/conformance/utils/tls"
+	"sigs.k8s.io/gateway-api/pkg/features"
+)
+
+func init() {
+	ConformanceTests = append(ConformanceTests, BackendTLSPolicy)
+}
+
+var BackendTLSPolicy = suite.ConformanceTest{
+	ShortName:   "BackendTLSPolicy",
+	Description: "A single service that is targeted by a BackendTLSPolicy must successfully complete TLS termination",
+	Features: []features.FeatureName{
+		features.SupportGateway,
+		features.SupportHTTPRoute,
+		features.SupportBackendTLSPolicy,
+	},
+	Manifests: []string{"tests/backendtlspolicy.yaml"},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+		ns := "gateway-conformance-infra"
+		routeNN := types.NamespacedName{Name: "gateway-conformance-infra-test", Namespace: ns}
+		gwNN := types.NamespacedName{Name: "gateway-backendtlspolicy", Namespace: ns}
+
+		kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{ns})
+		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &gatewayv1.HTTPRoute{}, false, routeNN)
+		kubernetes.HTTPRouteMustHaveResolvedRefsConditionsTrue(t, suite.Client, suite.TimeoutConfig, routeNN, gwNN)
+
+		serverStr := "abc.example.com"
+
+		// Verify that the response to a backend-tls-only call to /backendTLS will return the matching SNI.
+		t.Run("Simple HTTP request targeting BackendTLSPolicy should reach infra-backend", func(t *testing.T) {
+			h.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr,
+				h.ExpectedResponse{
+					Namespace: ns,
+					Request: h.Request{
+						Host: serverStr,
+						Path: "/backendTLS",
+						SNI:  serverStr,
+					},
+					Response: h.Response{StatusCode: 200},
+				})
+		})
+
+		// For the re-encrypt case, we need to use the cert for the frontend tls listener.
+		certNN := types.NamespacedName{Name: "tls-checks-certificate", Namespace: ns}
+		cPem, keyPem, err := GetTLSSecret(suite.Client, certNN)
+		if err != nil {
+			t.Fatalf("unexpected error finding TLS secret: %v", err)
+		}
+		// Verify that the response to a re-encrypted call to /backendTLS will return the matching SNI.
+		t.Run("Re-encrypt HTTPS request targeting BackendTLSPolicy should reach infra-backend", func(t *testing.T) {
+			tls.MakeTLSRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, cPem, keyPem, serverStr,
+				h.ExpectedResponse{
+					Namespace: ns,
+					Request: h.Request{
+						Host: serverStr,
+						Path: "/backendTLS",
+						SNI:  serverStr,
+					},
+					Response: h.Response{StatusCode: 200},
+				})
+		})
+	},
+}

conformance/tests/backendtlspolicy.yaml

@@ -0,0 +1,153 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: gateway-backendtlspolicy
+  namespace: gateway-conformance-infra
+spec:
+  gatewayClassName: "{GATEWAY_CLASS_NAME}"
+  listeners:
+  - name: http
+    port: 80
+    protocol: HTTP
+    hostname: "abc.example.com"
+    allowedRoutes:
+      namespaces:
+        from: Same
+      kinds:
+      - kind: HTTPRoute
+  - name: https
+    port: 443
+    protocol: HTTPS
+    tls:
+      mode: Terminate
+      certificateRefs:
+      - group: ""
+        kind: Secret
+        name: tls-checks-certificate
+    hostname: "abc.example.com"
+    allowedRoutes:
+      namespaces:
+        from: Same
+      kinds:
+      - kind: HTTPRoute
+---
+apiVersion: gateway.networking.k8s.io/v1alpha3
+kind: BackendTLSPolicy
+metadata:
+  name: normative-test-backendtlspolicy
+  namespace: gateway-conformance-infra
+spec:
+  targetRefs:
+  - group: ""
+    kind: Service
+    name: "backendtlspolicy-test"
+    sectionName: "btls"
+  validation:
+    caCertificateRefs:
+    - group: ""
+      kind: ConfigMap
+      # This secret is generated dynamically by the test suite.
+      name: "backend-tls-checks-certificate"
+    hostname: "abc.example.com"
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: gateway-conformance-infra-test
+  namespace: gateway-conformance-infra
+spec:
+  parentRefs:
+  - name: gateway-backendtlspolicy
+    namespace: gateway-conformance-infra
+  hostnames:
+  - abc.example.com
+  rules:
+  - backendRefs:
+    - group: ""
+      kind: Service
+      name: backendtlspolicy-test
+      port: 443
+    matches:
+    - path:
+        type: Exact
+        value: /backendTLS
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: backendtlspolicy-test
+  namespace: gateway-conformance-infra
+spec:
+  selector:
+    app: backendtlspolicy-test
+  ports:
+  - name: "btls"
+    protocol: TCP
+    port: 443
+    targetPort: 8443
+---
+# Deployment must not be applied until after the secret is generated.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backendtlspolicy-test
+  namespace: gateway-conformance-infra
+  labels:
+    app: backendtlspolicy-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: backendtlspolicy-test
+  template:
+    metadata:
+      labels:
+        app: backendtlspolicy-test
+    spec:
+      containers:
+      - name: backendtlspolicy-test
+        image: gcr.io/k8s-staging-gateway-api/echo-basic:v20240412-v1.0.0-394-g40c666fd
+        volumeMounts:
+        - name: ca-volume
+          mountPath: /etc/ca-volume
+        - name: secret-volume
+          mountPath: /etc/secret-volume
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: CA_CERT
+          value: /etc/ca-volume/crt
+        - name: CA_CERT_KEY
+          value: /etc/ca-volume/key
+        - name: TLS_SERVER_CERT
+          value: /etc/secret-volume/crt
+        - name: TLS_SERVER_PRIVKEY
+          value: /etc/secret-volume/key
+        resources:
+          requests:
+            cpu: 10m
+      volumes:
+      - name: ca-volume
+        configMap:
+          # This configMap is generated dynamically by the test suite.
+          name: backend-tls-checks-certificate
+          items:
+          - key: ca.crt
+            path: crt
+          - key: key.crt
+            path: key
+      - name: secret-volume
+        secret:
+          # This secret is generated dynamically by the test suite.
+          secretName: tls-checks-certificate
+          items:
+          - key: tls.crt
+            path: crt
+          - key: tls.key
+            path: key

conformance/tests/grpcroute-exact-method-matching.go

@@ -46,7 +46,7 @@ var GRPCExactMethodMatching = suite.ConformanceTest{
 		ns := "gateway-conformance-infra"
 		routeNN := types.NamespacedName{Name: "exact-matching", Namespace: ns}
 		gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
-		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &v1.GRPCRoute{}, routeNN)
+		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &v1.GRPCRoute{}, true, routeNN)
 
 		testCases := []grpc.ExpectedResponse{
 			{

conformance/tests/grpcroute-header-matching.go

@@ -46,7 +46,7 @@ var GRPCRouteHeaderMatching = suite.ConformanceTest{
 		ns := "gateway-conformance-infra"
 		routeNN := types.NamespacedName{Name: "grpc-header-matching", Namespace: ns}
 		gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
-		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &v1.GRPCRoute{}, routeNN)
+		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &v1.GRPCRoute{}, true, routeNN)
 
 		testCases := []grpc.ExpectedResponse{{
 			EchoRequest: &pb.EchoRequest{},

conformance/tests/grpcroute-listener-hostname-matching.go

@@ -52,15 +52,15 @@ var GRPCRouteListenerHostnameMatching = suite.ConformanceTest{
 		gwNN := types.NamespacedName{Name: "grpcroute-listener-hostname-matching", Namespace: ns}
 
 		_ = kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName,
-			kubernetes.NewGatewayRef(gwNN, "listener-1"), &v1.GRPCRoute{},
+			kubernetes.NewGatewayRef(gwNN, "listener-1"), &v1.GRPCRoute{}, true,
 			types.NamespacedName{Namespace: ns, Name: "backend-v1"},
 		)
 		_ = kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName,
-			kubernetes.NewGatewayRef(gwNN, "listener-2"), &v1.GRPCRoute{},
+			kubernetes.NewGatewayRef(gwNN, "listener-2"), &v1.GRPCRoute{}, true,
 			types.NamespacedName{Namespace: ns, Name: "backend-v2"},
 		)
 		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName,
-			kubernetes.NewGatewayRef(gwNN, "listener-3", "listener-4"), &v1.GRPCRoute{},
+			kubernetes.NewGatewayRef(gwNN, "listener-3", "listener-4"), &v1.GRPCRoute{}, true,
 			types.NamespacedName{Namespace: ns, Name: "backend-v3"},
 		)
 

conformance/tests/grpcroute-named-rule.go

@@ -47,7 +47,7 @@ var GRPCRouteNamedRule = suite.ConformanceTest{
 		ns := "gateway-conformance-infra"
 		routeNN := types.NamespacedName{Name: "grpc-named-rules", Namespace: ns}
 		gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
-		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &v1.GRPCRoute{}, routeNN)
+		gwAddr := kubernetes.GatewayAndRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), &v1.GRPCRoute{}, true, routeNN)
 
 		testCases := []grpc.ExpectedResponse{
 			{

conformance/tests/httproute-weight.go

@@ -105,7 +105,7 @@ func testDistribution(t *testing.T, suite *suite.ConformanceTestSuite, gwAddr st
 			if err != nil {
 				return fmt.Errorf("failed to roundtrip request: %w", err)
 			}
-			if err := http.CompareRequest(t, &req, cReq, cRes, expected); err != nil {
+			if err := http.CompareRoundTrip(t, &req, cReq, cRes, expected); err != nil {
 				return fmt.Errorf("response expectation failed for request: %w", err)
 			}
 

conformance/tests/tlsroute-simple-same-namespace.go

@@ -49,7 +49,7 @@ var TLSRouteSimpleSameNamespace = suite.ConformanceTest{
 		ns := "gateway-conformance-infra"
 		routeNN := types.NamespacedName{Name: "gateway-conformance-infra-test", Namespace: ns}
 		gwNN := types.NamespacedName{Name: "gateway-tlsroute", Namespace: ns}
-		certNN := types.NamespacedName{Name: "tls-passthrough-checks-certificate", Namespace: ns}
+		certNN := types.NamespacedName{Name: "tls-checks-certificate", Namespace: ns}
 
 		kubernetes.NamespacesMustBeReady(t, suite.Client, suite.TimeoutConfig, []string{ns})