test: add multi gateway case (#98)

Signed-off-by: ashing <[email protected]>

Author: ronething

config/samples/config.yaml

@@ -1,4 +1,4 @@
-log_level: "debug"                               # The log level of the API7 Ingress Controller.
+log_level: "info"                               # The log level of the API7 Ingress Controller.
                                                 # the default value is "info".
 
 controller_name: gateway.api7.io/api7-ingress-controller  # The controller name of the API7 Ingress Controller,
@@ -15,16 +15,3 @@ leader_election:
   retry_period: 2s                      # retry_period is the time in seconds that the acting controller
                                         # will wait between tries of actions with the controller.
   disable: false                        # Whether to disable leader election.
-
-# ingress_class: api7 # The ingress class name of the API7 Ingress Controller.
-# ingress_publish_service: "" # The service name of the ingress publish service.
-# ingress_status_address: [] # The status address of the ingress.
-# gateway_configs:                  # The configuration of the API7 Gateway.
-# - name: api7                      # The name of the Gateway in the Gateway API.
-#   control_plane:
-#     admin_key: "${ADMIN_KEY}"     # The admin key of the control plane.
-#     endpoints:         
-#       - ${ENDPOINT}               # The endpoint of the control plane.
-#     tls_verify: false
-#   addresses:                      # record the status address of the gateway-api gateway
-#   - "172.18.0.4"                  # The LB IP of the gateway service.

internal/controller/config/config.go

@@ -89,26 +89,6 @@ func (c *Config) Validate() error {
 	return nil
 }
 
-//nolint:unused
-func (c *Config) validateGatewayConfig(gc *GatewayConfig) error {
-
-	if gc.Name == "" {
-		return fmt.Errorf("control_planesp[].gateway_name is required")
-	}
-	if gc.ControlPlane.AdminKey == "" {
-		return fmt.Errorf("control_planes[].admin_api.admin_key is required")
-	}
-	if len(gc.ControlPlane.Endpoints) == 0 {
-		return fmt.Errorf("control_planes[].admin_api.endpoints is required")
-	}
-	if gc.ControlPlane.TLSVerify == nil {
-		gc.ControlPlane.TLSVerify = new(bool)
-		*gc.ControlPlane.TLSVerify = true
-	}
-
-	return nil
-}
-
 func GetControllerName() string {
 	return ControllerConfig.ControllerName
 }

test/e2e/gatewayapi/httproute.go

@@ -33,7 +33,7 @@ spec:
           value: "%s"
 `
 
-	var defautlGatewayClass = `
+	var gatewayClassYaml = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: GatewayClass
 metadata:
@@ -42,7 +42,7 @@ spec:
   controllerName: %s
 `
 
-	var defautlGateway = `
+	var defaultGateway = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -59,7 +59,7 @@ spec:
       kind: GatewayProxy
       name: api7-proxy-config
 `
-	var defautlGatewayHTTPS = `
+	var defaultGatewayHTTPS = `
 apiVersion: gateway.networking.k8s.io/v1
 kind: Gateway
 metadata:
@@ -111,7 +111,7 @@ spec:
 
 		By("create GatewayClass")
 		gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix())
-		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "")
+		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "")
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
 		time.Sleep(5 * time.Second)
 
@@ -122,7 +122,7 @@ spec:
 		Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
 
 		By("create Gateway")
-		err = s.CreateResourceFromString(fmt.Sprintf(defautlGateway, gatewayClassName))
+		err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayClassName))
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 		time.Sleep(5 * time.Second)
 
@@ -144,7 +144,7 @@ spec:
 		createSecret(s, secretName)
 		By("create GatewayClass")
 		gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix())
-		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "")
+		err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "")
 		Expect(err).NotTo(HaveOccurred(), "creating GatewayClass")
 		time.Sleep(5 * time.Second)
 
@@ -155,7 +155,7 @@ spec:
 		Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message")
 
 		By("create Gateway")
-		err = s.CreateResourceFromString(fmt.Sprintf(defautlGatewayHTTPS, gatewayClassName))
+		err = s.CreateResourceFromString(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName))
 		Expect(err).NotTo(HaveOccurred(), "creating Gateway")
 		time.Sleep(5 * time.Second)
 
@@ -211,6 +211,157 @@ spec:
 		})
 	})
 
+	Context("HTTPRoute with Multiple Gateway", func() {
+		var additionalGatewayGroupID string
+		var additionalNamespace string
+		var additionalGatewayClassName string
+
+		var additionalGatewayProxyYaml = `
+apiVersion: gateway.apisix.io/v1alpha1
+kind: GatewayProxy
+metadata:
+  name: additional-proxy-config
+spec:
+  provider:
+    type: ControlPlane
+    controlPlane:
+      endpoints:
+      - %s
+      auth:
+        type: AdminKey
+        adminKey:
+          value: "%s"
+`
+
+		var additionalGateway = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: additional-gateway
+spec:
+  gatewayClassName: %s
+  listeners:
+    - name: http-additional
+      protocol: HTTP
+      port: 80
+      allowedRoutes:
+        namespaces:
+          from: All
+  infrastructure:
+    parametersRef:
+      group: gateway.apisix.io
+      kind: GatewayProxy
+      name: additional-proxy-config
+`
+
+		// HTTPRoute that references both gateways
+		var multiGatewayHTTPRoute = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: multi-gateway-route
+spec:
+  parentRefs:
+  - name: api7ee
+    namespace: %s
+  - name: additional-gateway
+    namespace: %s
+  hostnames:
+  - httpbin.example
+  - httpbin-additional.example
+  rules:
+  - matches: 
+    - path:
+        type: Exact
+        value: /get
+    backendRefs:
+    - name: httpbin-service-e2e-test
+      port: 80
+`
+
+		BeforeEach(func() {
+			beforeEachHTTP()
+
+			By("Create additional gateway group")
+			var err error
+			additionalGatewayGroupID, additionalNamespace, err = s.CreateAdditionalGatewayGroup("multi-gw")
+			Expect(err).NotTo(HaveOccurred(), "creating additional gateway group")
+
+			By("Create additional GatewayProxy")
+			// Get admin key for the additional gateway group
+			resources, exists := s.GetAdditionalGatewayGroup(additionalGatewayGroupID)
+			Expect(exists).To(BeTrue(), "additional gateway group should exist")
+
+			By("Create additional GatewayClass")
+			additionalGatewayClassName = fmt.Sprintf("api7-%d", time.Now().Unix())
+			err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, additionalGatewayClassName, s.GetControllerName()), "")
+			Expect(err).NotTo(HaveOccurred(), "creating additional GatewayClass")
+			time.Sleep(5 * time.Second)
+			By("Check additional GatewayClass condition")
+			gcyaml, err := s.GetResourceYaml("GatewayClass", additionalGatewayClassName)
+			Expect(err).NotTo(HaveOccurred(), "getting additional GatewayClass yaml")
+			Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking additional GatewayClass condition status")
+			Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking additional GatewayClass condition message")
+
+			additionalGatewayProxy := fmt.Sprintf(additionalGatewayProxyYaml, framework.DashboardTLSEndpoint, resources.AdminAPIKey)
+			err = s.CreateResourceFromStringWithNamespace(additionalGatewayProxy, additionalNamespace)
+			Expect(err).NotTo(HaveOccurred(), "creating additional GatewayProxy")
+
+			By("Create additional Gateway")
+			err = s.CreateResourceFromStringWithNamespace(
+				fmt.Sprintf(additionalGateway, additionalGatewayClassName),
+				additionalNamespace,
+			)
+			Expect(err).NotTo(HaveOccurred(), "creating additional Gateway")
+			time.Sleep(5 * time.Second)
+		})
+
+		It("HTTPRoute should be accessible through both gateways", func() {
+			By("Create HTTPRoute referencing both gateways")
+			multiGatewayRoute := fmt.Sprintf(multiGatewayHTTPRoute, s.Namespace(), additionalNamespace)
+			ResourceApplied("HTTPRoute", "multi-gateway-route", multiGatewayRoute, 1)
+
+			By("Access through default gateway")
+			s.NewAPISIXClient().
+				GET("/get").
+				WithHost("httpbin.example").
+				Expect().
+				Status(http.StatusOK)
+
+			By("Access through additional gateway")
+			client, err := s.NewAPISIXClientForGatewayGroup(additionalGatewayGroupID)
+			Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway")
+
+			client.
+				GET("/get").
+				WithHost("httpbin-additional.example").
+				Expect().
+				Status(http.StatusOK)
+
+			By("Delete Additional Gateway")
+			err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, additionalGatewayClassName), additionalNamespace)
+			Expect(err).NotTo(HaveOccurred(), "deleting additional Gateway")
+			time.Sleep(5 * time.Second)
+
+			By("HTTPRoute should still be accessible through default gateway")
+			s.NewAPISIXClient().
+				GET("/get").
+				WithHost("httpbin.example").
+				Expect().
+				Status(http.StatusOK)
+
+			By("HTTPRoute should not be accessible through additional gateway")
+			client, err = s.NewAPISIXClientForGatewayGroup(additionalGatewayGroupID)
+			Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway")
+
+			client.
+				GET("/get").
+				WithHost("httpbin-additional.example").
+				Expect().
+				Status(http.StatusNotFound)
+		})
+	})
+
 	Context("HTTPRoute Base", func() {
 		var exactRouteByGet = `
 apiVersion: gateway.networking.k8s.io/v1

test/e2e/scaffold/dp.go

@@ -15,7 +15,6 @@
 package scaffold
 
 import (
-	"github.com/gruntwork-io/terratest/modules/k8s"
 	. "github.com/onsi/gomega"
 
 	"github.com/api7/api7-ingress-controller/test/e2e/framework"
@@ -43,36 +42,13 @@ func (s *Scaffold) deployDataplane() {
 }
 
 func (s *Scaffold) newAPISIXTunnels() error {
-	var (
-		httpNodePort  int
-		httpsNodePort int
-		httpPort      int
-		httpsPort     int
-		serviceName   = "api7ee3-apisix-gateway-mtls"
-	)
-
-	svc := s.dataplaneService
-	for _, port := range svc.Spec.Ports {
-		if port.Name == "http" {
-			httpNodePort = int(port.NodePort)
-			httpPort = int(port.Port)
-		} else if port.Name == "https" {
-			httpsNodePort = int(port.NodePort)
-			httpsPort = int(port.Port)
-		}
-	}
-	s.apisixHttpTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, serviceName,
-		httpNodePort, httpPort)
-	s.apisixHttpsTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, serviceName,
-		httpsNodePort, httpsPort)
-
-	if err := s.apisixHttpTunnel.ForwardPortE(s.t); err != nil {
+	serviceName := "api7ee3-apisix-gateway-mtls"
+	httpTunnel, httpsTunnel, err := s.createDataplaneTunnels(s.dataplaneService, s.kubectlOptions, serviceName)
+	if err != nil {
 		return err
 	}
-	s.addFinalizers(s.apisixHttpTunnel.Close)
-	if err := s.apisixHttpsTunnel.ForwardPortE(s.t); err != nil {
-		return err
-	}
-	s.addFinalizers(s.apisixHttpsTunnel.Close)
+
+	s.apisixHttpTunnel = httpTunnel
+	s.apisixHttpsTunnel = httpsTunnel
 	return nil
 }