From b7bb5e191e137ff883c4a0b3d0aa3f8180f9cdd9 Mon Sep 17 00:00:00 2001 From: ashing Date: Fri, 18 Apr 2025 11:58:07 +0800 Subject: [PATCH 1/5] fix: config Signed-off-by: ashing --- config/samples/config.yaml | 13 ------------- internal/controller/config/config.go | 20 -------------------- 2 files changed, 33 deletions(-) diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 8344ce041..d0463d096 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -15,16 +15,3 @@ leader_election: retry_period: 2s # retry_period is the time in seconds that the acting controller # will wait between tries of actions with the controller. disable: false # Whether to disable leader election. - -# ingress_class: api7 # The ingress class name of the API7 Ingress Controller. -# ingress_publish_service: "" # The service name of the ingress publish service. -# ingress_status_address: [] # The status address of the ingress. -# gateway_configs: # The configuration of the API7 Gateway. -# - name: api7 # The name of the Gateway in the Gateway API. -# control_plane: -# admin_key: "${ADMIN_KEY}" # The admin key of the control plane. -# endpoints: -# - ${ENDPOINT} # The endpoint of the control plane. -# tls_verify: false -# addresses: # record the status address of the gateway-api gateway -# - "172.18.0.4" # The LB IP of the gateway service. diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 0cd870ffe..d760a726a 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -89,26 +89,6 @@ func (c *Config) Validate() error { return nil } -//nolint:unused -func (c *Config) validateGatewayConfig(gc *GatewayConfig) error { - - if gc.Name == "" { - return fmt.Errorf("control_planesp[].gateway_name is required") - } - if gc.ControlPlane.AdminKey == "" { - return fmt.Errorf("control_planes[].admin_api.admin_key is required") - } - if len(gc.ControlPlane.Endpoints) == 0 { - return fmt.Errorf("control_planes[].admin_api.endpoints is required") - } - if gc.ControlPlane.TLSVerify == nil { - gc.ControlPlane.TLSVerify = new(bool) - *gc.ControlPlane.TLSVerify = true - } - - return nil -} - func GetControllerName() string { return ControllerConfig.ControllerName } From 51392c0739bdcc240e2c2fd809fe20fbaa22e9b4 Mon Sep 17 00:00:00 2001 From: ashing Date: Fri, 18 Apr 2025 17:04:55 +0800 Subject: [PATCH 2/5] fix: r Signed-off-by: ashing --- test/e2e/gatewayapi/httproute.go | 171 +++++++++++++++++ test/e2e/scaffold/dp.go | 36 +--- test/e2e/scaffold/scaffold.go | 302 +++++++++++++++++++++++++++++++ 3 files changed, 479 insertions(+), 30 deletions(-) diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index e0af23c35..27fb6be6d 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -211,6 +211,177 @@ spec: }) }) + Context("HTTPRoute with Multiple Gateway", func() { + var additionalGatewayGroupID string + var additionalNamespace string + + var additionalGatewayProxyYaml = ` +apiVersion: gateway.apisix.io/v1alpha1 +kind: GatewayProxy +metadata: + name: additional-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + endpoints: + - %s + auth: + type: AdminKey + adminKey: + value: "%s" +` + + var additionalGateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: additional-gateway +spec: + gatewayClassName: %s + listeners: + - name: http-additional + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All + infrastructure: + parametersRef: + group: gateway.apisix.io + kind: GatewayProxy + name: additional-proxy-config +` + + // HTTPRoute that references both gateways + var multiGatewayHTTPRoute = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: multi-gateway-route +spec: + parentRefs: + - name: api7ee + namespace: %s + - name: additional-gateway + namespace: %s + hostnames: + - httpbin.example + - httpbin-additional.example + rules: + - matches: + - path: + type: Exact + value: /get + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + + var gatewayClassName string + + BeforeEach(func() { + By("Create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create GatewayClass") + gatewayClassName = fmt.Sprintf("api7-%d", time.Now().Unix()) + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") + time.Sleep(5 * time.Second) + + By("check GatewayClass condition") + gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) + Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") + Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") + Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") + + By("create Gateway") + err = s.CreateResourceFromString(fmt.Sprintf(defautlGateway, gatewayClassName)) + Expect(err).NotTo(HaveOccurred(), "creating Gateway") + time.Sleep(5 * time.Second) + + By("check Gateway condition") + gwyaml, err := s.GetResourceYaml("Gateway", "api7ee") + Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") + Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") + Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the api7-ingress-controller"), "checking Gateway condition message") + + By("Create additional gateway group") + additionalGatewayGroupID, additionalNamespace, err = s.CreateAdditionalGatewayGroup("multi-gw") + Expect(err).NotTo(HaveOccurred(), "creating additional gateway group") + + By("Initialize dataplane client for additional gateway group") + err = s.InitializeDataplaneClientForGatewayGroup(additionalGatewayGroupID) + Expect(err).NotTo(HaveOccurred(), "initializing dataplane client") + + By("Create additional GatewayProxy") + // Get admin key for the additional gateway group + resources, exists := s.GetAdditionalGatewayGroup(additionalGatewayGroupID) + Expect(exists).To(BeTrue(), "additional gateway group should exist") + + additionalGatewayProxy := fmt.Sprintf(additionalGatewayProxyYaml, framework.DashboardTLSEndpoint, resources.AdminAPIKey) + err = s.CreateResourceFromStringWithNamespace(additionalGatewayProxy, additionalNamespace) + Expect(err).NotTo(HaveOccurred(), "creating additional GatewayProxy") + + By("Create additional Gateway") + err = s.CreateResourceFromStringWithNamespace( + fmt.Sprintf(additionalGateway, gatewayClassName), + additionalNamespace, + ) + Expect(err).NotTo(HaveOccurred(), "creating additional Gateway") + time.Sleep(5 * time.Second) + }) + + It("HTTPRoute should be accessible through both gateways", func() { + By("Create HTTPRoute referencing both gateways") + multiGatewayRoute := fmt.Sprintf(multiGatewayHTTPRoute, s.Namespace(), additionalNamespace) + ResourceApplied("HTTPRoute", "multi-gateway-route", multiGatewayRoute, 1) + + By("Access through default gateway") + s.NewAPISIXClient(). + GET("/get"). + WithHost("httpbin.example"). + Expect(). + Status(http.StatusOK) + + By("Access through additional gateway") + client, err := s.NewAPISIXClientForGatewayGroup(additionalGatewayGroupID) + Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") + + client. + GET("/get"). + WithHost("httpbin-additional.example"). + Expect(). + Status(http.StatusOK) + + By("Delete Additional Gateway") + err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, gatewayClassName), additionalNamespace) + Expect(err).NotTo(HaveOccurred(), "deleting additional Gateway") + time.Sleep(5 * time.Second) + + By("HTTPRoute should still be accessible through default gateway") + s.NewAPISIXClient(). + GET("/get"). + WithHost("httpbin.example"). + Expect(). + Status(http.StatusOK) + + By("HTTPRoute should not be accessible through additional gateway") + client, err = s.NewAPISIXClientForGatewayGroup(additionalGatewayGroupID) + Expect(err).NotTo(HaveOccurred(), "creating client for additional gateway") + + client. + GET("/get"). + WithHost("httpbin-additional.example"). + Expect(). + Status(http.StatusNotFound) + }) + }) + Context("HTTPRoute Base", func() { var exactRouteByGet = ` apiVersion: gateway.networking.k8s.io/v1 diff --git a/test/e2e/scaffold/dp.go b/test/e2e/scaffold/dp.go index 6476aa561..e55aca7c1 100644 --- a/test/e2e/scaffold/dp.go +++ b/test/e2e/scaffold/dp.go @@ -15,7 +15,6 @@ package scaffold import ( - "github.com/gruntwork-io/terratest/modules/k8s" . "github.com/onsi/gomega" "github.com/api7/api7-ingress-controller/test/e2e/framework" @@ -43,36 +42,13 @@ func (s *Scaffold) deployDataplane() { } func (s *Scaffold) newAPISIXTunnels() error { - var ( - httpNodePort int - httpsNodePort int - httpPort int - httpsPort int - serviceName = "api7ee3-apisix-gateway-mtls" - ) - - svc := s.dataplaneService - for _, port := range svc.Spec.Ports { - if port.Name == "http" { - httpNodePort = int(port.NodePort) - httpPort = int(port.Port) - } else if port.Name == "https" { - httpsNodePort = int(port.NodePort) - httpsPort = int(port.Port) - } - } - s.apisixHttpTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, serviceName, - httpNodePort, httpPort) - s.apisixHttpsTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, serviceName, - httpsNodePort, httpsPort) - - if err := s.apisixHttpTunnel.ForwardPortE(s.t); err != nil { + serviceName := "api7ee3-apisix-gateway-mtls" + httpTunnel, httpsTunnel, err := s.createDataplaneTunnels(s.dataplaneService, s.kubectlOptions, serviceName) + if err != nil { return err } - s.addFinalizers(s.apisixHttpTunnel.Close) - if err := s.apisixHttpsTunnel.ForwardPortE(s.t); err != nil { - return err - } - s.addFinalizers(s.apisixHttpsTunnel.Close) + + s.apisixHttpTunnel = httpTunnel + s.apisixHttpsTunnel = httpsTunnel return nil } diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index e46a8a4f0..fdc9b8a07 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -97,6 +97,18 @@ type Scaffold struct { apisixUDPTunnel *k8s.Tunnel // apisixControlTunnel *k8s.Tunnel + // Support for multiple Gateway groups + additionalGatewayGroups map[string]*GatewayGroupResources +} + +// GatewayGroupResources contains resources associated with a specific Gateway group +type GatewayGroupResources struct { + GatewayGroupID string + Namespace string + DataplaneService *corev1.Service + HttpTunnel *k8s.Tunnel + HttpsTunnel *k8s.Tunnel + AdminAPIKey string } func (s *Scaffold) AdminKey() string { @@ -371,6 +383,9 @@ func (s *Scaffold) beforeEach() { s.label["apisix.ingress.watch"] = s.namespace } + // Initialize additionalGatewayGroups map + s.additionalGatewayGroups = make(map[string]*GatewayGroupResources) + var nsLabel map[string]string if !s.opts.DisableNamespaceLabel { nsLabel = s.label @@ -450,6 +465,11 @@ func (s *Scaffold) afterEach() { defer GinkgoRecover() s.DeleteGatewayGroup(s.gatewaygroupid) + // Clean up all additional gateway groups + for gatewayGroupID := range s.additionalGatewayGroups { + s.DeleteGatewayGroup(gatewayGroupID) + } + if CurrentSpecReport().Failed() { if os.Getenv("TEST_ENV") == "CI" { _, _ = fmt.Fprintln(GinkgoWriter, "Dumping namespace contents") @@ -463,6 +483,17 @@ func (s *Scaffold) afterEach() { } } + // Delete all additional namespaces + for _, resources := range s.additionalGatewayGroups { + err := k8s.DeleteNamespaceE(s.t, &k8s.KubectlOptions{ + ConfigPath: s.opts.Kubeconfig, + Namespace: resources.Namespace, + }, resources.Namespace) + if err != nil { + s.Logf("failed to delete additional namespace %s: %v", resources.Namespace, err) + } + } + // if the test case is successful, just delete namespace err := k8s.DeleteNamespaceE(s.t, s.kubectlOptions, s.namespace) Expect(err).NotTo(HaveOccurred(), "deleting namespace "+s.namespace) @@ -576,3 +607,274 @@ func (s *Scaffold) labelSelector(label string) metav1.ListOptions { func (s *Scaffold) GetControllerName() string { return s.opts.ControllerName } + +// CreateAdditionalGatewayGroup creates a new gateway group and deploys a dataplane for it. +// It returns the gateway group ID and namespace name where the dataplane is deployed. +func (s *Scaffold) CreateAdditionalGatewayGroup(namePrefix string) (string, string, error) { + // Create a new namespace for this gateway group + additionalNS := fmt.Sprintf("%s-%d", namePrefix, time.Now().Unix()) + + // Create namespace with the same labels + var nsLabel map[string]string + if !s.opts.DisableNamespaceLabel { + nsLabel = s.label + } + k8s.CreateNamespaceWithMetadata(s.t, s.kubectlOptions, metav1.ObjectMeta{Name: additionalNS, Labels: nsLabel}) + + // Create new kubectl options for the new namespace + kubectlOpts := &k8s.KubectlOptions{ + ConfigPath: s.opts.Kubeconfig, + Namespace: additionalNS, + } + + // Create a new gateway group + gatewayGroupID := s.CreateNewGatewayGroupWithIngress() + s.Logf("additional gateway group id: %s in namespace %s", gatewayGroupID, additionalNS) + + // Get the admin key for this gateway group + adminKey := s.GetAdminKey(gatewayGroupID) + s.Logf("additional gateway group admin api key: %s", adminKey) + + // Store gateway group info + resources := &GatewayGroupResources{ + GatewayGroupID: gatewayGroupID, + Namespace: additionalNS, + AdminAPIKey: adminKey, + } + + serviceName := fmt.Sprintf("api7ee3-apisix-gateway-%s", namePrefix) + + // Deploy dataplane for this gateway group + svc := s.DeployGateway(framework.DataPlaneDeployOptions{ + GatewayGroupID: gatewayGroupID, + Namespace: additionalNS, + Name: serviceName, + ServiceName: serviceName, + DPManagerEndpoint: framework.DPManagerTLSEndpoint, + SetEnv: true, + SSLKey: framework.TestKey, + SSLCert: framework.TestCert, + TLSEnabled: true, + ForIngressGatewayGroup: true, + ServiceHTTPPort: 9080, + ServiceHTTPSPort: 9443, + }) + + resources.DataplaneService = svc + + // Create tunnels for the dataplane + httpTunnel, httpsTunnel, err := s.createDataplaneTunnels(svc, kubectlOpts, serviceName) + if err != nil { + return "", "", err + } + + resources.HttpTunnel = httpTunnel + resources.HttpsTunnel = httpsTunnel + + // Store in the map + s.additionalGatewayGroups[gatewayGroupID] = resources + + return gatewayGroupID, additionalNS, nil +} + +// createDataplaneTunnels creates HTTP and HTTPS tunnels for a dataplane service. +// It's extracted from newAPISIXTunnels to be reusable for additional gateway groups. +func (s *Scaffold) createDataplaneTunnels( + svc *corev1.Service, + kubectlOpts *k8s.KubectlOptions, + serviceName string, +) (*k8s.Tunnel, *k8s.Tunnel, error) { + var ( + httpNodePort int + httpsNodePort int + httpPort int + httpsPort int + ) + + for _, port := range svc.Spec.Ports { + if port.Name == "http" { + httpNodePort = int(port.NodePort) + httpPort = int(port.Port) + } else if port.Name == "https" { + httpsNodePort = int(port.NodePort) + httpsPort = int(port.Port) + } + } + + httpTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, + httpNodePort, httpPort) + httpsTunnel := k8s.NewTunnel(kubectlOpts, k8s.ResourceTypeService, serviceName, + httpsNodePort, httpsPort) + + if err := httpTunnel.ForwardPortE(s.t); err != nil { + return nil, nil, err + } + s.addFinalizers(httpTunnel.Close) + + if err := httpsTunnel.ForwardPortE(s.t); err != nil { + httpTunnel.Close() + return nil, nil, err + } + s.addFinalizers(httpsTunnel.Close) + + return httpTunnel, httpsTunnel, nil +} + +// GetAdditionalGatewayGroup returns resources associated with a specific Gateway group +func (s *Scaffold) GetAdditionalGatewayGroup(gatewayGroupID string) (*GatewayGroupResources, bool) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + return resources, exists +} + +// NewAPISIXClientForGatewayGroup creates an HTTP client for a specific Gateway group +func (s *Scaffold) NewAPISIXClientForGatewayGroup(gatewayGroupID string) (*httpexpect.Expect, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + u := url.URL{ + Scheme: "http", + Host: resources.HttpTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{}, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }), nil +} + +// NewAPISIXHttpsClientForGatewayGroup creates an HTTPS client for a specific Gateway group +func (s *Scaffold) NewAPISIXHttpsClientForGatewayGroup(gatewayGroupID string, host string) (*httpexpect.Expect, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + u := url.URL{ + Scheme: "https", + Host: resources.HttpsTunnel.Endpoint(), + } + return httpexpect.WithConfig(httpexpect.Config{ + BaseURL: u.String(), + Client: &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + // accept any certificate; for testing only! + InsecureSkipVerify: true, + ServerName: host, + }, + }, + }, + Reporter: httpexpect.NewAssertReporter( + httpexpect.NewAssertReporter(GinkgoT()), + ), + }), nil +} + +// CleanupAdditionalGatewayGroup cleans up resources associated with a specific Gateway group +func (s *Scaffold) CleanupAdditionalGatewayGroup(gatewayGroupID string) error { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + // Delete the gateway group + s.DeleteGatewayGroup(gatewayGroupID) + + // Delete the namespace + err := k8s.DeleteNamespaceE(s.t, &k8s.KubectlOptions{ + ConfigPath: s.opts.Kubeconfig, + Namespace: resources.Namespace, + }, resources.Namespace) + + // Remove from the map + delete(s.additionalGatewayGroups, gatewayGroupID) + + return err +} + +// InitializeDataplaneClientForGatewayGroup initializes the Dashboard client for a specific Gateway group +func (s *Scaffold) InitializeDataplaneClientForGatewayGroup(gatewayGroupID string) error { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + // HTTP client + httpURL := fmt.Sprintf("http://%s/apisix/admin", resources.HttpTunnel.Endpoint()) + clusterName := fmt.Sprintf("gateway-%s", gatewayGroupID[:8]) // Use first 8 chars of gateway group ID as identifier + + err := s.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ + Name: clusterName, + ControllerName: s.opts.ControllerName, + Labels: map[string]string{"k8s/controller-name": s.opts.ControllerName}, + BaseURL: httpURL, + AdminKey: resources.AdminAPIKey, + }) + if err != nil { + return err + } + + // HTTPS client + httpsURL := fmt.Sprintf("https://%s/apisix/admin", resources.HttpsTunnel.Endpoint()) + clusterNameHTTPS := fmt.Sprintf("gateway-%s-https", gatewayGroupID[:8]) + + err = s.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ + Name: clusterNameHTTPS, + BaseURL: httpsURL, + AdminKey: resources.AdminAPIKey, + SkipTLSVerify: true, + }) + + return err +} + +// GetDataplaneResourceForGatewayGroup returns the dataplane resource for a specific Gateway group +func (s *Scaffold) GetDataplaneResourceForGatewayGroup(gatewayGroupID string) (dashboard.Cluster, error) { + _, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + clusterName := fmt.Sprintf("gateway-%s", gatewayGroupID[:8]) + return s.apisixCli.Cluster(clusterName), nil +} + +// GetDataplaneResourceHTTPSForGatewayGroup returns the HTTPS dataplane resource for a specific Gateway group +func (s *Scaffold) GetDataplaneResourceHTTPSForGatewayGroup(gatewayGroupID string) (dashboard.Cluster, error) { + _, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + clusterName := fmt.Sprintf("gateway-%s-https", gatewayGroupID[:8]) + return s.apisixCli.Cluster(clusterName), nil +} + +// GetGatewayGroupHTTPEndpoint returns the HTTP endpoint for a specific Gateway group +func (s *Scaffold) GetGatewayGroupHTTPEndpoint(gatewayGroupID string) (string, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return "", fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + return resources.HttpTunnel.Endpoint(), nil +} + +// GetGatewayGroupHTTPSEndpoint returns the HTTPS endpoint for a specific Gateway group +func (s *Scaffold) GetGatewayGroupHTTPSEndpoint(gatewayGroupID string) (string, error) { + resources, exists := s.additionalGatewayGroups[gatewayGroupID] + if !exists { + return "", fmt.Errorf("gateway group %s not found", gatewayGroupID) + } + + return resources.HttpsTunnel.Endpoint(), nil +} From 4c09d814b65874098eadd7a06e7ecc92ad768ce0 Mon Sep 17 00:00:00 2001 From: ashing Date: Fri, 18 Apr 2025 17:27:24 +0800 Subject: [PATCH 3/5] fix: r Signed-off-by: ashing --- test/e2e/gatewayapi/httproute.go | 66 +++++++++++--------------------- test/e2e/scaffold/scaffold.go | 58 ---------------------------- 2 files changed, 23 insertions(+), 101 deletions(-) diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index 27fb6be6d..2adb0d465 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -33,7 +33,7 @@ spec: value: "%s" ` - var defautlGatewayClass = ` + var gatewayClassYaml = ` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -42,7 +42,7 @@ spec: controllerName: %s ` - var defautlGateway = ` + var defaultGateway = ` apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -59,7 +59,7 @@ spec: kind: GatewayProxy name: api7-proxy-config ` - var defautlGatewayHTTPS = ` + var defaultGatewayHTTPS = ` apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -111,7 +111,7 @@ spec: By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -122,7 +122,7 @@ spec: Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("create Gateway") - err = s.CreateResourceFromString(fmt.Sprintf(defautlGateway, gatewayClassName)) + err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayClassName)) Expect(err).NotTo(HaveOccurred(), "creating Gateway") time.Sleep(5 * time.Second) @@ -144,7 +144,7 @@ spec: createSecret(s, secretName) By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -155,7 +155,7 @@ spec: Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("create Gateway") - err = s.CreateResourceFromString(fmt.Sprintf(defautlGatewayHTTPS, gatewayClassName)) + err = s.CreateResourceFromString(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName)) Expect(err).NotTo(HaveOccurred(), "creating Gateway") time.Sleep(5 * time.Second) @@ -214,6 +214,7 @@ spec: Context("HTTPRoute with Multiple Gateway", func() { var additionalGatewayGroupID string var additionalNamespace string + var additionalGatewayClassName string var additionalGatewayProxyYaml = ` apiVersion: gateway.apisix.io/v1alpha1 @@ -278,58 +279,37 @@ spec: port: 80 ` - var gatewayClassName string - BeforeEach(func() { - By("Create GatewayProxy") - gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.DashboardTLSEndpoint, s.AdminKey()) - err := s.CreateResourceFromString(gatewayProxy) - Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") - time.Sleep(5 * time.Second) - - By("create GatewayClass") - gatewayClassName = fmt.Sprintf("api7-%d", time.Now().Unix()) - err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") - Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") - time.Sleep(5 * time.Second) - - By("check GatewayClass condition") - gcyaml, err := s.GetResourceYaml("GatewayClass", gatewayClassName) - Expect(err).NotTo(HaveOccurred(), "getting GatewayClass yaml") - Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") - Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") - - By("create Gateway") - err = s.CreateResourceFromString(fmt.Sprintf(defautlGateway, gatewayClassName)) - Expect(err).NotTo(HaveOccurred(), "creating Gateway") - time.Sleep(5 * time.Second) - - By("check Gateway condition") - gwyaml, err := s.GetResourceYaml("Gateway", "api7ee") - Expect(err).NotTo(HaveOccurred(), "getting Gateway yaml") - Expect(gwyaml).To(ContainSubstring(`status: "True"`), "checking Gateway condition status") - Expect(gwyaml).To(ContainSubstring("message: the gateway has been accepted by the api7-ingress-controller"), "checking Gateway condition message") + beforeEachHTTP() By("Create additional gateway group") + var err error additionalGatewayGroupID, additionalNamespace, err = s.CreateAdditionalGatewayGroup("multi-gw") Expect(err).NotTo(HaveOccurred(), "creating additional gateway group") - By("Initialize dataplane client for additional gateway group") - err = s.InitializeDataplaneClientForGatewayGroup(additionalGatewayGroupID) - Expect(err).NotTo(HaveOccurred(), "initializing dataplane client") - By("Create additional GatewayProxy") // Get admin key for the additional gateway group resources, exists := s.GetAdditionalGatewayGroup(additionalGatewayGroupID) Expect(exists).To(BeTrue(), "additional gateway group should exist") + By("Create additional GatewayClass") + additionalGatewayClassName = fmt.Sprintf("api7-%d", time.Now().Unix()) + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(gatewayClassYaml, additionalGatewayClassName, s.GetControllerName()), "") + Expect(err).NotTo(HaveOccurred(), "creating additional GatewayClass") + time.Sleep(5 * time.Second) + By("Check additional GatewayClass condition") + gcyaml, err := s.GetResourceYaml("GatewayClass", additionalGatewayClassName) + Expect(err).NotTo(HaveOccurred(), "getting additional GatewayClass yaml") + Expect(gcyaml).To(ContainSubstring(`status: "True"`), "checking additional GatewayClass condition status") + Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking additional GatewayClass condition message") + additionalGatewayProxy := fmt.Sprintf(additionalGatewayProxyYaml, framework.DashboardTLSEndpoint, resources.AdminAPIKey) err = s.CreateResourceFromStringWithNamespace(additionalGatewayProxy, additionalNamespace) Expect(err).NotTo(HaveOccurred(), "creating additional GatewayProxy") By("Create additional Gateway") err = s.CreateResourceFromStringWithNamespace( - fmt.Sprintf(additionalGateway, gatewayClassName), + fmt.Sprintf(additionalGateway, additionalGatewayClassName), additionalNamespace, ) Expect(err).NotTo(HaveOccurred(), "creating additional Gateway") @@ -359,7 +339,7 @@ spec: Status(http.StatusOK) By("Delete Additional Gateway") - err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, gatewayClassName), additionalNamespace) + err = s.DeleteResourceFromStringWithNamespace(fmt.Sprintf(additionalGateway, additionalGatewayClassName), additionalNamespace) Expect(err).NotTo(HaveOccurred(), "deleting additional Gateway") time.Sleep(5 * time.Second) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index fdc9b8a07..d8ce90e1c 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -801,64 +801,6 @@ func (s *Scaffold) CleanupAdditionalGatewayGroup(gatewayGroupID string) error { return err } -// InitializeDataplaneClientForGatewayGroup initializes the Dashboard client for a specific Gateway group -func (s *Scaffold) InitializeDataplaneClientForGatewayGroup(gatewayGroupID string) error { - resources, exists := s.additionalGatewayGroups[gatewayGroupID] - if !exists { - return fmt.Errorf("gateway group %s not found", gatewayGroupID) - } - - // HTTP client - httpURL := fmt.Sprintf("http://%s/apisix/admin", resources.HttpTunnel.Endpoint()) - clusterName := fmt.Sprintf("gateway-%s", gatewayGroupID[:8]) // Use first 8 chars of gateway group ID as identifier - - err := s.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ - Name: clusterName, - ControllerName: s.opts.ControllerName, - Labels: map[string]string{"k8s/controller-name": s.opts.ControllerName}, - BaseURL: httpURL, - AdminKey: resources.AdminAPIKey, - }) - if err != nil { - return err - } - - // HTTPS client - httpsURL := fmt.Sprintf("https://%s/apisix/admin", resources.HttpsTunnel.Endpoint()) - clusterNameHTTPS := fmt.Sprintf("gateway-%s-https", gatewayGroupID[:8]) - - err = s.apisixCli.AddCluster(context.Background(), &dashboard.ClusterOptions{ - Name: clusterNameHTTPS, - BaseURL: httpsURL, - AdminKey: resources.AdminAPIKey, - SkipTLSVerify: true, - }) - - return err -} - -// GetDataplaneResourceForGatewayGroup returns the dataplane resource for a specific Gateway group -func (s *Scaffold) GetDataplaneResourceForGatewayGroup(gatewayGroupID string) (dashboard.Cluster, error) { - _, exists := s.additionalGatewayGroups[gatewayGroupID] - if !exists { - return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) - } - - clusterName := fmt.Sprintf("gateway-%s", gatewayGroupID[:8]) - return s.apisixCli.Cluster(clusterName), nil -} - -// GetDataplaneResourceHTTPSForGatewayGroup returns the HTTPS dataplane resource for a specific Gateway group -func (s *Scaffold) GetDataplaneResourceHTTPSForGatewayGroup(gatewayGroupID string) (dashboard.Cluster, error) { - _, exists := s.additionalGatewayGroups[gatewayGroupID] - if !exists { - return nil, fmt.Errorf("gateway group %s not found", gatewayGroupID) - } - - clusterName := fmt.Sprintf("gateway-%s-https", gatewayGroupID[:8]) - return s.apisixCli.Cluster(clusterName), nil -} - // GetGatewayGroupHTTPEndpoint returns the HTTP endpoint for a specific Gateway group func (s *Scaffold) GetGatewayGroupHTTPEndpoint(gatewayGroupID string) (string, error) { resources, exists := s.additionalGatewayGroups[gatewayGroupID] From df3027a435c9e053ded648b76135fbd4fadaa648 Mon Sep 17 00:00:00 2001 From: ashing Date: Fri, 18 Apr 2025 17:36:25 +0800 Subject: [PATCH 4/5] fix: r Signed-off-by: ashing --- config/samples/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/samples/config.yaml b/config/samples/config.yaml index d0463d096..1858b3660 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -1,4 +1,4 @@ -log_level: "debug" # The log level of the API7 Ingress Controller. +log_level: "info" # The log level of the API7 Ingress Controller. # the default value is "info". controller_name: gateway.api7.io/api7-ingress-controller # The controller name of the API7 Ingress Controller, From c7026c5d4a4df6c724ea8bc9def9791367e838ad Mon Sep 17 00:00:00 2001 From: ashing Date: Fri, 18 Apr 2025 17:53:11 +0800 Subject: [PATCH 5/5] fix: r Signed-off-by: ashing --- test/e2e/scaffold/scaffold.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index d8ce90e1c..f73123867 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -465,11 +465,6 @@ func (s *Scaffold) afterEach() { defer GinkgoRecover() s.DeleteGatewayGroup(s.gatewaygroupid) - // Clean up all additional gateway groups - for gatewayGroupID := range s.additionalGatewayGroups { - s.DeleteGatewayGroup(gatewayGroupID) - } - if CurrentSpecReport().Failed() { if os.Getenv("TEST_ENV") == "CI" { _, _ = fmt.Fprintln(GinkgoWriter, "Dumping namespace contents") @@ -485,13 +480,8 @@ func (s *Scaffold) afterEach() { // Delete all additional namespaces for _, resources := range s.additionalGatewayGroups { - err := k8s.DeleteNamespaceE(s.t, &k8s.KubectlOptions{ - ConfigPath: s.opts.Kubeconfig, - Namespace: resources.Namespace, - }, resources.Namespace) - if err != nil { - s.Logf("failed to delete additional namespace %s: %v", resources.Namespace, err) - } + err := s.CleanupAdditionalGatewayGroup(resources.GatewayGroupID) + Expect(err).NotTo(HaveOccurred(), "cleaning up additional gateway group") } // if the test case is successful, just delete namespace