diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 8344ce041..1858b3660 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -1,4 +1,4 @@ -log_level: "debug" # The log level of the API7 Ingress Controller. +log_level: "info" # The log level of the API7 Ingress Controller. # the default value is "info". controller_name: gateway.api7.io/api7-ingress-controller # The controller name of the API7 Ingress Controller, @@ -15,16 +15,3 @@ leader_election: retry_period: 2s # retry_period is the time in seconds that the acting controller # will wait between tries of actions with the controller. disable: false # Whether to disable leader election. - -# ingress_class: api7 # The ingress class name of the API7 Ingress Controller. -# ingress_publish_service: "" # The service name of the ingress publish service. -# ingress_status_address: [] # The status address of the ingress. -# gateway_configs: # The configuration of the API7 Gateway. -# - name: api7 # The name of the Gateway in the Gateway API. -# control_plane: -# admin_key: "${ADMIN_KEY}" # The admin key of the control plane. -# endpoints: -# - ${ENDPOINT} # The endpoint of the control plane. -# tls_verify: false -# addresses: # record the status address of the gateway-api gateway -# - "172.18.0.4" # The LB IP of the gateway service. diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 0cd870ffe..d760a726a 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -89,26 +89,6 @@ func (c *Config) Validate() error { return nil } -//nolint:unused -func (c *Config) validateGatewayConfig(gc *GatewayConfig) error { - - if gc.Name == "" { - return fmt.Errorf("control_planesp[].gateway_name is required") - } - if gc.ControlPlane.AdminKey == "" { - return fmt.Errorf("control_planes[].admin_api.admin_key is required") - } - if len(gc.ControlPlane.Endpoints) == 0 { - return fmt.Errorf("control_planes[].admin_api.endpoints is required") - } - if gc.ControlPlane.TLSVerify == nil { - gc.ControlPlane.TLSVerify = new(bool) - *gc.ControlPlane.TLSVerify = true - } - - return nil -} - func GetControllerName() string { return ControllerConfig.ControllerName } diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index e0af23c35..2adb0d465 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -33,7 +33,7 @@ spec: value: "%s" ` - var defautlGatewayClass = ` + var gatewayClassYaml = ` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -42,7 +42,7 @@ spec: controllerName: %s ` - var defautlGateway = ` + var defaultGateway = ` apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -59,7 +59,7 @@ spec: kind: GatewayProxy name: api7-proxy-config ` - var defautlGatewayHTTPS = ` + var defaultGatewayHTTPS = ` apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -111,7 +111,7 @@ spec: By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -122,7 +122,7 @@ spec: Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("create Gateway") - err = s.CreateResourceFromString(fmt.Sprintf(defautlGateway, gatewayClassName)) + err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayClassName)) Expect(err).NotTo(HaveOccurred(), "creating Gateway") time.Sleep(5 * time.Second) @@ -144,7 +144,7 @@ spec: createSecret(s, secretName) By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -155,7 +155,7 @@ spec: Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("create Gateway") - err = s.CreateResourceFromString(fmt.Sprintf(defautlGatewayHTTPS, gatewayClassName)) + err = s.CreateResourceFromString(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName)) Expect(err).NotTo(HaveOccurred(), "creating Gateway") time.Sleep(5 * time.Second) @@ -211,6 +211,157 @@ spec: }) }) + Context("HTTPRoute with Multiple Gateway", func() { + var additionalGatewayGroupID string + var additionalNamespace string + var additionalGatewayClassName string + + var additionalGatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: additional-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + + var additionalGateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: additional-gateway +spec: + gatewayClassName: %s + listeners: + - name: http-additional + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: additional-proxy-config +` + + // HTTPRoute that references both gateways + var multiGatewayHTTPRoute = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: multi-gateway-route +spec: + parentRefs: + - name: api7ee + namespace: %s + - name: additional-gateway + namespace: %s + hostnames: + - httpbin.example + - httpbin-additional.example + rules: + - matches: + - path: + type: Exact + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + BeforeEach(func() { + beforeEachHTTP() + + By("Create additional gateway group") + var err error + additionalGatewayGroupID, additionalNamespace, err = s.CreateAdditionalGatewayGroup("multi-gw") + Expect(err).NotTo(HaveOccurred(), "creating additional gateway group") + + By("Create additional GatewayProxy") + // Get admin key for the additional gateway group + resources, exists := s.GetAdditionalGatewayGroup(additionalGatewayGroupID) + Expect(exists).To(BeTrue(), "additional gateway group should exist") + + By("Create additional GatewayClass") + additionalGatewayClassName = fmt.Sprintf("api7-%d", time.Now().Unix()) + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, additionalGatewayClassName, s.GetControllerName()), "") + Expect(err).NotTo(HaveOccurred(), "creating additional GatewayClass") + time.Sleep(5 * time.Second) + By("Check additional GatewayClass condition") + gcyaml, err := s.GetResourceYaml("GatewayClass", additionalGatewayClassName) + Expect(err).NotTo(HaveOccurred(), "getting additional GatewayClass yaml") + Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking additional GatewayClass condition status") + Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking additional GatewayClass condition message") + + additionalGatewayProxy := fmt.Sprintf(additionalGatewayProxyYaml, framework.DashboardTLSEndpoint, resources.AdminAPIKey) + err = s.CreateResourceFromStringWithNamespace(additionalGatewayProxy, additionalNamespace) + Expect(err).NotTo(HaveOccurred(), "creating additional GatewayProxy") + + By("Create additional Gateway") + err = s.CreateResourceFromStringWithNamespace( + fmt.Sprintf(additionalGateway, additionalGatewayClassName), + additionalNamespace, + ) + Expect(err).NotTo(HaveOccurred(), "creating additional Gateway") + time.Sleep(5 * time.Second) + }) + + It("HTTPRoute should be accessible through both gateways", func() { + By("Create HTTPRoute referencing both gateways") + multiGatewayRoute := fmt.Sprintf(multiGatewayHTTPRoute, s.Namespace(), additionalNamespace) + ResourceApplied("HTTPRoute", "multi-gateway-route", multiGatewayRoute, 1) + + By("Access through default gateway") + s.NewAPISIXClient(). + GET("/get"). + WithHost("httpbin.example"). + Expect(). + Status(http.StatusOK) + + By("Access through additional gateway") + client, err := s.NewAPISIXClientForGatewayGroup(additionalGatewayGroupID) + Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") + + client. + GET("/get"). + WithHost("httpbin-additional.example"). + Expect(). + Status(http.StatusOK) + + By("Delete Additional Gateway") + err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, additionalGatewayClassName), additionalNamespace) + Expect(err).NotTo(HaveOccurred(), "deleting additional Gateway") + time.Sleep(5 * time.Second) + + By("HTTPRoute should still be accessible through default gateway") + s.NewAPISIXClient(). + GET("/get"). + WithHost("httpbin.example"). + Expect(). + Status(http.StatusOK) + + By("HTTPRoute should not be accessible through additional gateway") + client, err = s.NewAPISIXClientForGatewayGroup(additionalGatewayGroupID) + Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") + + client. + GET("/get"). + WithHost("httpbin-additional.example"). + Expect(). + Status(http.StatusNotFound) + }) + }) + Context("HTTPRoute Base", func() { var exactRouteByGet = ` apiVersion: gateway.networking.k8s.io/v1 diff --git a/test/e2e/scaffold/dp.go b/test/e2e/scaffold/dp.go index 6476aa561..e55aca7c1 100644 --- a/test/e2e/scaffold/dp.go +++ b/test/e2e/scaffold/dp.go @@ -15,7 +15,6 @@ package scaffold import ( - "github.com/gruntwork-io/terratest/modules/k8s" . "github.com/onsi/gomega" "github.com/api7/api7-ingress-controller/test/e2e/framework" @@ -43,36 +42,13 @@ func (s *Scaffold) deployDataplane() { } func (s *Scaffold) newAPISIXTunnels() error { - var ( - httpNodePort int - httpsNodePort int - httpPort int - httpsPort int - serviceName = "api7ee3-apisix-gateway-mtls" - ) - - svc := s.dataplaneService - for _, port := range svc.Spec.Ports { - if port.Name == "http" { - httpNodePort = int(port.NodePort) - httpPort = int(port.Port) - } else if port.Name == "https" { - httpsNodePort = int(port.NodePort) - httpsPort = int(port.Port) - } - } - s.apisixHttpTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, serviceName, - httpNodePort, httpPort) - s.apisixHttpsTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, serviceName, - httpsNodePort, httpsPort) - - if err := s.apisixHttpTunnel.ForwardPortE(s.t); err != nil { + serviceName := "api7ee3-apisix-gateway-mtls" + httpTunnel, httpsTunnel, err := s.createDataplaneTunnels(s.dataplaneService, s.kubectlOptions, serviceName) + if err != nil { return err } - s.addFinalizers(s.apisixHttpTunnel.Close) - if err := s.apisixHttpsTunnel.ForwardPortE(s.t); err != nil { - return err - } - s.addFinalizers(s.apisixHttpsTunnel.Close) + + s.apisixHttpTunnel = httpTunnel + s.apisixHttpsTunnel = httpsTunnel return nil } diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index e46a8a4f0..f73123867 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -97,6 +97,18 @@ type Scaffold struct { apisixUDPTunnel *k8s.Tunnel // apisixControlTunnel *k8s.Tunnel + // Support for multiple Gateway groups + additionalGatewayGroups map[string]*GatewayGroupResources +} + +// GatewayGroupResources contains resources associated with a specific Gateway group +type GatewayGroupResources struct { + GatewayGroupID string + Namespace string + DataplaneService *corev1.Service + HttpTunnel *k8s.Tunnel + HttpsTunnel *k8s.Tunnel + AdminAPIKey string } func (s *Scaffold) AdminKey() string { @@ -371,6 +383,9 @@ func (s *Scaffold) beforeEach() { s.label["apisix.ingress.watch"] = s.namespace } + // Initialize additionalGatewayGroups map + s.additionalGatewayGroups = make(map[string]*GatewayGroupResources) + var nsLabel map[string]string if !s.opts.DisableNamespaceLabel { nsLabel = s.label @@ -463,6 +478,12 @@ func (s *Scaffold) afterEach() { } } + // Delete all additional namespaces + for _, resources := range s.additionalGatewayGroups { + err := s.CleanupAdditionalGatewayGroup(resources.GatewayGroupID) + Expect(err).NotTo(HaveOccurred(), "cleaning up additional gateway group") + } + // if the test case is successful, just delete namespace err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace) Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace) @@ -576,3 +597,216 @@ func (s *Scaffold) labelSelector(label string) metav1.ListOptions { func (s *Scaffold) GetControllerName() string { return s.opts.ControllerName } + +// CreateAdditionalGatewayGroup creates a new gateway group and deploys a dataplane for it. +// It returns the gateway group ID and namespace name where the dataplane is deployed. +func (s *Scaffold) CreateAdditionalGatewayGroup(namePrefix string) (string, string, error) { + // Create a new namespace for this gateway group + additionalNS := fmt.Sprintf("%s-%d", namePrefix, time.Now().Unix()) + + // Create namespace with the same labels + var nsLabel map[string]string + if !s.opts.DisableNamespaceLabel { + nsLabel = s.label + } + k8s.CreateNamespaceWithMetadata(s.t, s.kubectlOptions, metav1.ObjectMeta{Name: additionalNS, Labels: nsLabel}) + + // Create new kubectl options for the new namespace + kubectlOpts := &k8s.KubectlOptions{ + ConfigPath: s.opts.Kubeconfig, + Namespace: additionalNS, + } + + // Create a new gateway group + gatewayGroupID := s.CreateNewGatewayGroupWithIngress() + s.Logf("additional gateway group id: %s in namespace %s", gatewayGroupID, additionalNS) + + // Get the admin key for this gateway group + adminKey := s.GetAdminKey(gatewayGroupID) + s.Logf("additional gateway group admin api key: %s", adminKey) + + // Store gateway group info + resources := &GatewayGroupResources{ + GatewayGroupID: gatewayGroupID, + Namespace: additionalNS, + AdminAPIKey: adminKey, + } + + serviceName := fmt.Sprintf("api7ee3-apisix-gateway-%s", namePrefix) + + // Deploy dataplane for this gateway group + svc := s.DeployGateway(framework.DataPlaneDeployOptions{ + GatewayGroupID: gatewayGroupID, + Namespace: additionalNS, + Name: serviceName, + ServiceName: serviceName, + DPManagerEndpoint: framework.DPManagerTLSEndpoint, + SetEnv: true, + SSLKey: framework.TestKey, + SSLCert: framework.TestCert, + TLSEnabled: true, + ForIngressGatewayGroup: true, + ServiceHTTPPort: 9080, + ServiceHTTPSPort: 9443, + }) + + resources.DataplaneService = svc + + // Create tunnels for the dataplane + httpTunnel, httpsTunnel, err := s.createDataplaneTunnels(svc, kubectlOpts, serviceName) + if err != nil { + return "", "", err + } + + resources.HttpTunnel = httpTunnel + resources.HttpsTunnel = httpsTunnel + + // Store in the map + s.additionalGatewayGroups[gatewayGroupID] = resources + + return gatewayGroupID, additionalNS, nil +} + +// createDataplaneTunnels creates HTTP and HTTPS tunnels for a dataplane service. +// It's extracted from newAPISIXTunnels to be reusable for additional gateway groups. +func (s *Scaffold) createDataplaneTunnels( + svc *corev1.Service, + kubectlOpts *k8s.KubectlOptions, + serviceName string, +) (*k8s.Tunnel, *k8s.Tunnel, error) { + var ( + httpNodePort int + httpsNodePort int + httpPort int + httpsPort int + ) + + for _, port := range svc.Spec.Ports { + if port.Name == "http" { + httpNodePort = int(port.NodePort) + httpPort = int(port.Port) + } else if port.Name == "https" { + httpsNodePort = int(port.NodePort) + httpsPort = int(port.Port) + } + } + + httpTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, + httpNodePort, httpPort) + httpsTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, + httpsNodePort, httpsPort) + + if err := httpTunnel.ForwardPortE(s.t); err != nil { + return nil, nil, err + } + s.addFinalizers(httpTunnel.Close) + + if err := httpsTunnel.ForwardPortE(s.t); err != nil { + httpTunnel.Close() + return nil, nil, err + } + s.addFinalizers(httpsTunnel.Close) + + return httpTunnel, httpsTunnel, nil +} + +// GetAdditionalGatewayGroup returns resources associated with a specific Gateway group +func (s *Scaffold) GetAdditionalGatewayGroup(gatewayGroupID string) (*GatewayGroupResources, bool) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + return resources, exists +} + +// NewAPISIXClientForGatewayGroup creates an HTTP client for a specific Gateway group +func (s *Scaffold) NewAPISIXClientForGatewayGroup(gatewayGroupID string) (*httpexpect.Expect, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + u := url.URL{ + Scheme: "http", + Host: resources.HttpTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{}, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }), nil +} + +// NewAPISIXHttpsClientForGatewayGroup creates an HTTPS client for a specific Gateway group +func (s *Scaffold) NewAPISIXHttpsClientForGatewayGroup(gatewayGroupID string, host string) (*httpexpect.Expect, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + u := url.URL{ + Scheme: "https", + Host: resources.HttpsTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // accept any certificate; for testing only! + InsecureSkipVerify: true, + ServerName: host, + }, + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }), nil +} + +// CleanupAdditionalGatewayGroup cleans up resources associated with a specific Gateway group +func (s *Scaffold) CleanupAdditionalGatewayGroup(gatewayGroupID string) error { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + // Delete the gateway group + s.DeleteGatewayGroup(gatewayGroupID) + + // Delete the namespace + err := k8s.DeleteNamespaceE(s.t, &k8s.KubectlOptions{ + ConfigPath: s.opts.Kubeconfig, + Namespace: resources.Namespace, + }, resources.Namespace) + + // Remove from the map + delete(s.additionalGatewayGroups, gatewayGroupID) + + return err +} + +// GetGatewayGroupHTTPEndpoint returns the HTTP endpoint for a specific Gateway group +func (s *Scaffold) GetGatewayGroupHTTPEndpoint(gatewayGroupID string) (string, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return "", fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + return resources.HttpTunnel.Endpoint(), nil +} + +// GetGatewayGroupHTTPSEndpoint returns the HTTPS endpoint for a specific Gateway group +func (s *Scaffold) GetGatewayGroupHTTPSEndpoint(gatewayGroupID string) (string, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return "", fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + return resources.HttpsTunnel.Endpoint(), nil +}