From d1fbb6490b43d8d0cdab479bd9977dc26d2f3c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=82=9F=E7=A9=BA?= Date: Fri, 11 Jul 2025 15:16:04 +0800 Subject: [PATCH 1/2] feat: Add load test for 2000 ApisixRoutes --- test/e2e/load-test/e2e_test.go | 44 ++++++++ test/e2e/load-test/spec_subject.go | 160 +++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+) create mode 100644 test/e2e/load-test/e2e_test.go create mode 100644 test/e2e/load-test/spec_subject.go diff --git a/test/e2e/load-test/e2e_test.go b/test/e2e/load-test/e2e_test.go new file mode 100644 index 0000000000..2898b76c17 --- /dev/null +++ b/test/e2e/load-test/e2e_test.go @@ -0,0 +1,44 @@ +package load_test + +import ( + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/api7/gopkg/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var closer io.Closer + +func init() { + // save log locally + file, err := os.Create(time.Now().Format("load_test_200601021504.log")) + if err != nil { + log.Fatalf("failed to create log file, err: %v", err) + } + closer = file + GinkgoWriter.TeeTo(file) +} + +// Run long-term-stability tests using Ginkgo runner. +func TestLongTermStability(t *testing.T) { + defer func() { _ = closer.Close() }() + + RegisterFailHandler(Fail) + var f = framework.NewFramework() + _ = f + + scaffold.NewDeployer = func(s *scaffold.Scaffold) scaffold.Deployer { + return scaffold.NewAPISIXDeployer(s) + } + + _, _ = fmt.Fprintf(GinkgoWriter, "Starting load-test suite\n") + RunSpecs(t, "long-term-stability suite") +} diff --git a/test/e2e/load-test/spec_subject.go b/test/e2e/load-test/spec_subject.go new file mode 100644 index 0000000000..689e6ef01a --- /dev/null +++ b/test/e2e/load-test/spec_subject.go @@ -0,0 +1,160 @@ +package load + +import ( + "bytes" + "fmt" + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +const gatewayProxyYaml = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: GatewayProxy +metadata: + name: apisix-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + service: + name: %s + port: 9180 + auth: + type: AdminKey + adminKey: + value: "%s" +` + +const ingressClassYaml = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: apisix +spec: + controller: "apisix.apache.org/apisix-ingress-controller" + parameters: + apiGroup: "apisix.apache.org" + kind: "GatewayProxy" + name: "apisix-proxy-config" + namespace: %s + scope: "Namespace" +` + +var _ = Describe("Load Test", func() { + var ( + s = scaffold.NewScaffold(&scaffold.Options{ + ControllerName: "apisix.apache.org/apisix-ingress-controller", + }) + ) + + BeforeEach(func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey()) + err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create IngressClass") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, s.Namespace()), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass") + time.Sleep(5 * time.Second) + }) + + Context("Load Test 2000 ApisixRoute", func() { + It("test 2000 ApisixRoute", func() { + const total = 1000 + + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /* + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + + By(fmt.Sprintf("prepare %d ApisixRoutes", total)) + var text = bytes.NewBuffer(nil) + for i := range total { + name := getRouteName(i) + text.WriteString(fmt.Sprintf(apisixRouteSpec, name, name)) + text.WriteString("\n---\n") + } + + err := s.CreateResourceFromString(text.String()) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoutes") + + By("count time") + now := time.Now() + time.Sleep(30 * time.Second) + + var c = make(chan int, total) + for i := range total { + c <- i + } + + var totalWorks = 0 + for { + if len(c) == 0 { + close(c) + break + } + i := <-c + name := getRouteName(i) + By(fmt.Sprintf("[%d/%d]try to verify %s", totalWorks, total, name)) + if s.NewAPISIXClient().GET("/get").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode == http.StatusOK { + totalWorks++ + By(fmt.Sprintf("[%d/%d]%s works", totalWorks, total, name)) + continue + } + time.Sleep(100 * time.Millisecond) + c <- i + } + + // w := sync.WaitGroup{} + // for i := range total { + // time.Sleep(100 * time.Millisecond) + // name := getRouteName(i) + // w.Add(1) + // task := func(name string) { + // defer w.Done() + // By(fmt.Sprintf("to check ApisixRoute %s works", name)) + // err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + // resp := s.NewAPISIXClient().GET("/get").WithHeader("X-Route-Name", name).Expect().Raw() + // return resp.StatusCode == http.StatusOK, nil + // }) + // Expect(err).NotTo(HaveOccurred()) + // By(fmt.Sprintf("ApisixRoute %s works", name)) + // } + // go task(name) + // } + // + // w.Wait() + fmt.Printf("======2000 ApisixRoutes 生效时间为: %s =========", time.Since(now)) + }) + }) +}) + +func getRouteName(i int) string { + return fmt.Sprintf("test-route-%04d", i) +} From 5405ced435c0b9b681aa72cbe442a81b1d9ff07d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=82=9F=E7=A9=BA?= Date: Fri, 11 Jul 2025 16:57:27 +0800 Subject: [PATCH 2/2] feat: Increase memory and CPU limits for ingress, update load test to 2000 ApisixRoutes, and add test for ApisixRoute update time doc: recommended to use apisix-standalone mode for installation. (#2470) Signed-off-by: ashing --- docs/en/latest/getting-started.md | 10 +- docs/en/latest/upgrade-guide.md | 6 + internal/controller/httproute_controller.go | 2 +- internal/manager/run.go | 31 ---- internal/provider/adc/cache/indexer.go | 10 +- test/e2e/apisix/route.go | 36 +++++ test/e2e/framework/apisix_consts.go | 3 +- test/e2e/framework/manifests/apisix.yaml | 18 +++ test/e2e/framework/manifests/ingress.yaml | 8 +- test/e2e/gatewayapi/httproute.go | 96 ++++++++++++ test/e2e/load-test/e2e_test.go | 17 ++ test/e2e/load-test/spec_subject.go | 163 ++++++++++++++------ test/e2e/scaffold/apisix_deployer.go | 12 +- test/e2e/scaffold/scaffold.go | 33 ++++ 14 files changed, 351 insertions(+), 94 deletions(-) diff --git a/docs/en/latest/getting-started.md b/docs/en/latest/getting-started.md index 3f20bc2c5b..fc8d02137b 100644 --- a/docs/en/latest/getting-started.md +++ b/docs/en/latest/getting-started.md @@ -43,9 +43,9 @@ Before installing APISIX Ingress Controller, ensure you have: 1. A working Kubernetes cluster (version 1.26+) 2. [Helm](https://helm.sh/) (version 3.8+) installed -### Install APISIX and APISIX Ingress Controller +### Install APISIX and APISIX Ingress Controller (Standalone API-driven mode) -Install the Gateway API CRDs, APISIX, and APISIX Ingress Controller using the following commands: +Install the Gateway API CRDs, [APISIX Standalone API-driven mode](https://apisix.apache.org/docs/apisix/deployment-modes/#api-driven-experimental), and APISIX Ingress Controller using the following commands: ```bash helm repo add apisix https://charts.apiseven.com @@ -55,7 +55,11 @@ helm repo update helm install apisix \ --namespace ingress-apisix \ --create-namespace \ + --set apisix.deployment.role=traditional \ + --set apisix.deployment.role_traditional.config_provider=yaml \ + --set etcd.enabled=false \ --set ingress-controller.enabled=true \ + --set ingress-controller.config.provider.type=apisix-standalone \ --set ingress-controller.apisix.adminService.namespace=ingress-apisix \ --set ingress-controller.gatewayProxy.createDefault=true \ apisix/apisix @@ -66,7 +70,7 @@ helm install apisix \ Install the httpbin example application to test the configuration: ```bash -https://raw.githubusercontent.com/apache/apisix-ingress-controller/refs/heads/v2.0.0/examples/httpbin/deployment.yaml +kubectl apply -f https://raw.githubusercontent.com/apache/apisix-ingress-controller/refs/heads/v2.0.0/examples/httpbin/deployment.yaml ``` ### Configure a Route diff --git a/docs/en/latest/upgrade-guide.md b/docs/en/latest/upgrade-guide.md index 07473e2ee7..343fd366f1 100644 --- a/docs/en/latest/upgrade-guide.md +++ b/docs/en/latest/upgrade-guide.md @@ -66,6 +66,12 @@ etcdserver: In 2.0.0, all data plane configurations must originate from the Ingress Controller. Configurations via Admin API or any external methods are no longer supported and will be ignored or may cause errors. +#### APISIX With Etcd (Admin API) synchronization performance + +In APISIX Ingress Controller 2.0.0, ADC performs scheduled resource synchronization by comparing resources against the admin API response. + +Because the Admin API fills in default values, the submitted content may differ from the returned result. This breaks the diff, triggering full updates to data plane resources, causing cache invalidation and significant performance impact. + ### Ingress Configuration Changes #### Configuration Path Changes diff --git a/internal/controller/httproute_controller.go b/internal/controller/httproute_controller.go index 3a21e6a227..50b6e3b2ba 100644 --- a/internal/controller/httproute_controller.go +++ b/internal/controller/httproute_controller.go @@ -501,7 +501,7 @@ func (r *HTTPRouteReconciler) processHTTPRouteBackendRefs(tctx *provider.Transla if service.Spec.Type == corev1.ServiceTypeExternalName { tctx.Services[targetNN] = &service - return nil + continue } portExists := false diff --git a/internal/manager/run.go b/internal/manager/run.go index 54ee31c8bb..b48746c162 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -21,7 +21,6 @@ import ( "context" "crypto/tls" "os" - "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -172,36 +171,6 @@ func Run(ctx context.Context, logger logr.Logger) error { return err } - go func() { - setupLog.Info("starting provider sync") - initalSyncDelay := config.ControllerConfig.ProviderConfig.InitSyncDelay.Duration - time.AfterFunc(initalSyncDelay, func() { - setupLog.Info("trying to initialize provider") - if err := provider.Sync(ctx); err != nil { - setupLog.Error(err, "unable to sync resources to provider") - return - } - }) - - syncPeriod := config.ControllerConfig.ProviderConfig.SyncPeriod.Duration - if syncPeriod < 1 { - return - } - ticker := time.NewTicker(syncPeriod) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := provider.Sync(ctx); err != nil { - setupLog.Error(err, "unable to sync resources to provider") - return - } - case <-ctx.Done(): - return - } - } - }() - setupLog.Info("check ReferenceGrants is enabled") _, err = mgr.GetRESTMapper().KindsFor(schema.GroupVersionResource{ Group: v1beta1.GroupVersion.Group, diff --git a/internal/provider/adc/cache/indexer.go b/internal/provider/adc/cache/indexer.go index e19410b848..ad9dda3bd0 100644 --- a/internal/provider/adc/cache/indexer.go +++ b/internal/provider/adc/cache/indexer.go @@ -56,6 +56,12 @@ type LabelIndexer struct { GetLabels func(obj any) map[string]string } +// ref: https://pkg.go.dev/github.com/hashicorp/go-memdb#Txn.Get +// by adding suffixes to avoid prefix matching +func (emi *LabelIndexer) genKey(labelValues []string) []byte { + return []byte(strings.Join(labelValues, "/") + "\x00") +} + func (emi *LabelIndexer) FromObject(obj any) (bool, []byte, error) { labels := emi.GetLabels(obj) var labelValues []string @@ -69,7 +75,7 @@ func (emi *LabelIndexer) FromObject(obj any) (bool, []byte, error) { return false, nil, nil } - return true, []byte(strings.Join(labelValues, "/")), nil + return true, emi.genKey(labelValues), nil } func (emi *LabelIndexer) FromArgs(args ...any) ([]byte, error) { @@ -86,5 +92,5 @@ func (emi *LabelIndexer) FromArgs(args ...any) ([]byte, error) { labelValues = append(labelValues, value) } - return []byte(strings.Join(labelValues, "/")), nil + return emi.genKey(labelValues), nil } diff --git a/test/e2e/apisix/route.go b/test/e2e/apisix/route.go index a47e6a312a..e39e9c861d 100644 --- a/test/e2e/apisix/route.go +++ b/test/e2e/apisix/route.go @@ -356,6 +356,42 @@ spec: applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "httpbin-service-e2e-test"}, new(apiv2.ApisixUpstream), apisixUpstreamSpec1) Eventually(request).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) }) + + It("Multiple ApisixRoute with same prefix name", func() { + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + hosts: + - %s + paths: + - /* + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + By("apply ApisixRoute") + var apisixRoute apiv2.ApisixRoute + for _, id := range []string{"11111", "1111", "111", "11", "1"} { + name := fmt.Sprintf("route-%s", id) + host := fmt.Sprintf("httpbin-%s", id) + applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: name}, &apisixRoute, fmt.Sprintf(apisixRouteSpec, name, host)) + } + + By("verify ApisixRoute works") + for _, id := range []string{"1", "11", "111", "1111", "11111"} { + host := fmt.Sprintf("httpbin-%s", id) + Eventually(func() int { + return s.NewAPISIXClient().GET("/get").WithHost(host).Expect().Raw().StatusCode + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(Equal(http.StatusOK)) + } + }) }) Context("Test ApisixRoute reference ApisixUpstream", func() { diff --git a/test/e2e/framework/apisix_consts.go b/test/e2e/framework/apisix_consts.go index 42214982bb..a7ff0607c7 100644 --- a/test/e2e/framework/apisix_consts.go +++ b/test/e2e/framework/apisix_consts.go @@ -27,7 +27,8 @@ import ( ) var ( - ProviderType = cmp.Or(os.Getenv("PROVIDER_TYPE"), "apisix") + ProviderType = cmp.Or(os.Getenv("PROVIDER_TYPE"), "apisix") + ProviderSyncPeriod = cmp.Or(os.Getenv("PROVIDER_SYNC_PERIOD"), "200ms") ) var ( diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index affa4bfb57..568bde3df7 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -90,6 +90,9 @@ spec: - name: admin containerPort: 9180 protocol: TCP + - name: control + containerPort: 9090 + protocol: TCP volumeMounts: - name: config-writable mountPath: /usr/local/apisix/conf @@ -123,3 +126,18 @@ spec: selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} +--- +apiVersion: v1 +kind: Service +metadata: + name: apisix-control-api + labels: + app.kubernetes.io/name: apisix-control-api +spec: + ports: + - port: 9090 + name: control + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/name: apisix diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index c4bb101470..0bedfe142d 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -408,11 +408,11 @@ spec: periodSeconds: 10 resources: limits: - cpu: 500m - memory: 128Mi + cpu: "2" + memory: 1024Mi requests: - cpu: 10m - memory: 64Mi + cpu: "2" + memory: 1024Mi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index 6ac6f2622f..a634447e9e 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -1711,6 +1711,102 @@ spec: }) }) + Context("Test HTTPRoute Load Balancing", func() { + BeforeEach(beforeEachHTTP) + It("Test load balancing with ExternalName services", func() { + const servicesSpec = ` +apiVersion: v1 +kind: Service +metadata: + name: httpbin-external-domain +spec: + type: ExternalName + externalName: httpbin.org +--- +apiVersion: v1 +kind: Service +metadata: + name: mockapi7-external-domain +spec: + type: ExternalName + externalName: mock.api7.ai +--- +apiVersion: apisix.apache.org/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: passhost-node +spec: + targetRefs: + - name: httpbin-external-domain + kind: Service + group: "" + - name: mockapi7-external-domain + kind: Service + group: "" + passHost: node + scheme: https +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: lb-route +spec: + parentRefs: + - name: apisix + rules: + - matches: + - path: + type: Exact + value: /headers + backendRefs: + - name: httpbin-external-domain + port: 443 + weight: 1 + - name: mockapi7-external-domain + port: 443 + weight: 1 +` + + By("apply services and HTTPRoute") + err := s.CreateResourceFromString(servicesSpec) + Expect(err).ShouldNot(HaveOccurred(), "apply services and HTTPRoute") + time.Sleep(5 * time.Second) + + By("verify load balancing works") + // Test multiple requests to verify load balancing + upstreamHosts := make(map[string]int) + totalRequests := 20 + + for i := 0; i < totalRequests; i++ { + resp := s.NewAPISIXClient().GET("/headers").Expect().Status(http.StatusOK) + + // Parse JSON response to get the Host header + var responseBody map[string]any + resp.JSON().Decode(&responseBody) + + if headers, ok := responseBody["headers"].(map[string]any); ok { + var host string + if host, ok = headers["Host"].(string); !ok { + host, ok = headers["host"].(string) + } + if ok && host != "" { + upstreamHosts[host]++ + } + Expect(ok).To(BeTrue(), "Host header should be present") + Expect(host).Should(Or(Equal("httpbin.org"), Equal("mock.api7.ai"))) + } + time.Sleep(100 * time.Millisecond) // Small delay between requests + } + + By("verify both upstreams received requests") + Expect(upstreamHosts).Should(HaveLen(2)) + + for host, count := range upstreamHosts { + Expect(count).Should(BeNumerically(">", 0), fmt.Sprintf("upstream %s should receive requests", host)) + } + }) + }) + /* Context("HTTPRoute Status Updated", func() { }) diff --git a/test/e2e/load-test/e2e_test.go b/test/e2e/load-test/e2e_test.go index 2898b76c17..d28c30bf71 100644 --- a/test/e2e/load-test/e2e_test.go +++ b/test/e2e/load-test/e2e_test.go @@ -1,3 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package load_test import ( diff --git a/test/e2e/load-test/spec_subject.go b/test/e2e/load-test/spec_subject.go index 689e6ef01a..79de2cf5f3 100644 --- a/test/e2e/load-test/spec_subject.go +++ b/test/e2e/load-test/spec_subject.go @@ -1,13 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + package load import ( "bytes" + "context" "fmt" "net/http" "time" + "github.com/api7/gopkg/pkg/log" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/wait" "github.com/apache/apisix-ingress-controller/test/e2e/framework" "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" @@ -51,12 +72,14 @@ var _ = Describe("Load Test", func() { s = scaffold.NewScaffold(&scaffold.Options{ ControllerName: "apisix.apache.org/apisix-ingress-controller", }) + controlAPIClient scaffold.ControlAPIClient + err error ) BeforeEach(func() { By("create GatewayProxy") gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey()) - err := s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) + err = s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") time.Sleep(5 * time.Second) @@ -64,11 +87,15 @@ var _ = Describe("Load Test", func() { err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, s.Namespace()), "") Expect(err).NotTo(HaveOccurred(), "creating IngressClass") time.Sleep(5 * time.Second) + + By("port-forward to control api service") + controlAPIClient, err = s.ControlAPIClient() + Expect(err).NotTo(HaveOccurred(), "create control api client") }) Context("Load Test 2000 ApisixRoute", func() { It("test 2000 ApisixRoute", func() { - const total = 1000 + const total = 2000 const apisixRouteSpec = ` apiVersion: apisix.apache.org/v2 @@ -81,7 +108,7 @@ spec: - name: rule0 match: paths: - - /* + - /get exprs: - subject: scope: Header @@ -97,60 +124,74 @@ spec: var text = bytes.NewBuffer(nil) for i := range total { name := getRouteName(i) - text.WriteString(fmt.Sprintf(apisixRouteSpec, name, name)) + _, err := fmt.Fprintf(text, apisixRouteSpec, name, name) + Expect(err).NotTo(HaveOccurred()) text.WriteString("\n---\n") } - err := s.CreateResourceFromString(text.String()) Expect(err).NotTo(HaveOccurred(), "creating ApisixRoutes") - By("count time") - now := time.Now() - time.Sleep(30 * time.Second) - - var c = make(chan int, total) - for i := range total { - c <- i - } - - var totalWorks = 0 - for { - if len(c) == 0 { - close(c) - break + var ( + results []TestResult + now = time.Now() + ) + By("Test the time required for applying a large number of ApisixRoutes to take effect") + var times int + err = wait.PollUntilContextTimeout(context.Background(), time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + times++ + results, _, err := controlAPIClient.ListServices() + if err != nil { + log.Errorw("failed to ListServices", zap.Error(err)) + return false, nil } - i := <-c - name := getRouteName(i) - By(fmt.Sprintf("[%d/%d]try to verify %s", totalWorks, total, name)) - if s.NewAPISIXClient().GET("/get").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode == http.StatusOK { - totalWorks++ - By(fmt.Sprintf("[%d/%d]%s works", totalWorks, total, name)) - continue + if len(results) != total { + log.Debugw("number of effective services", zap.Int("number", len(results)), zap.Int("times", times)) + return false, nil } - time.Sleep(100 * time.Millisecond) - c <- i - } - - // w := sync.WaitGroup{} - // for i := range total { - // time.Sleep(100 * time.Millisecond) - // name := getRouteName(i) - // w.Add(1) - // task := func(name string) { - // defer w.Done() - // By(fmt.Sprintf("to check ApisixRoute %s works", name)) - // err := wait.PollUntilContextTimeout(context.Background(), 3*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { - // resp := s.NewAPISIXClient().GET("/get").WithHeader("X-Route-Name", name).Expect().Raw() - // return resp.StatusCode == http.StatusOK, nil - // }) - // Expect(err).NotTo(HaveOccurred()) - // By(fmt.Sprintf("ApisixRoute %s works", name)) - // } - // go task(name) - // } - // - // w.Wait() - fmt.Printf("======2000 ApisixRoutes 生效时间为: %s =========", time.Since(now)) + return len(results) == total, nil + }) + Expect(err).ShouldNot(HaveOccurred()) + results = append(results, TestResult{ + CaseName: fmt.Sprintf("Apply %d ApisixRoutes", total), + CostTime: time.Since(now), + }) + + By("Test the time required for an ApisixRoute update to take effect") + var apisixRouteSpec0 = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /headers + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + name := getRouteName(10) + err = s.CreateResourceFromString(fmt.Sprintf(apisixRouteSpec0, name, name)) + Expect(err).NotTo(HaveOccurred()) + now = time.Now() + Eventually(func() int { + return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode + }).WithTimeout(time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) + results = append(results, TestResult{ + CaseName: fmt.Sprintf("Update a single ApisixRoute base on %d ApisixRoutes", total), + CostTime: time.Since(now), + }) + + PrintResults(results) }) }) }) @@ -158,3 +199,25 @@ spec: func getRouteName(i int) string { return fmt.Sprintf("test-route-%04d", i) } + +type TestResult struct { + CaseName string + CostTime time.Duration +} + +func (tr TestResult) String() string { + return fmt.Sprintf("%s takes effect for %s", tr.CaseName, tr.CostTime) +} + +func PrintResults(results []TestResult) { + fmt.Printf("\n======================TEST RESULT ProviderSyncPeriod %s===============================\n", framework.ProviderSyncPeriod) + fmt.Printf("%-70s", "Test Case") + fmt.Printf("%-70s\n", "Time Required") + fmt.Printf("%-70s\n", "--------------------------------------------------------------------------------------") + for _, result := range results { + fmt.Printf("%-70s", result.CaseName) + fmt.Printf("%-70s\n", result.CostTime) + } + fmt.Println("======================================================================================") + fmt.Println() +} diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index b6fe7febcf..b5240cd9c0 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -260,7 +260,7 @@ func (s *APISIXDeployer) DeployIngress() { s.Framework.DeployIngress(framework.IngressDeployOpts{ ControllerName: s.opts.ControllerName, ProviderType: framework.ProviderType, - ProviderSyncPeriod: 200 * time.Millisecond, + ProviderSyncPeriod: getProviderSyncPeriod(), Namespace: s.namespace, Replicas: 1, }) @@ -270,12 +270,20 @@ func (s *APISIXDeployer) ScaleIngress(replicas int) { s.Framework.DeployIngress(framework.IngressDeployOpts{ ControllerName: s.opts.ControllerName, ProviderType: framework.ProviderType, - ProviderSyncPeriod: 200 * time.Millisecond, + ProviderSyncPeriod: getProviderSyncPeriod(), Namespace: s.namespace, Replicas: replicas, }) } +func getProviderSyncPeriod() time.Duration { + providerSyncPeriod, err := time.ParseDuration(framework.ProviderSyncPeriod) + if err != nil { + providerSyncPeriod = 200 * time.Millisecond + } + return providerSyncPeriod +} + // getEnvOrDefault returns environment variable value or default func getEnvOrDefault(key, defaultValue string) string { if value := os.Getenv(key); value != "" { diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index fe533a82cf..790a33eddc 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -20,6 +20,7 @@ package scaffold import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "net/url" @@ -31,6 +32,7 @@ import ( "github.com/gruntwork-io/terratest/modules/testing" . "github.com/onsi/ginkgo/v2" //nolint:staticcheck . "github.com/onsi/gomega" //nolint:staticcheck + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -425,6 +427,18 @@ func (s *Scaffold) KubeOpts() *k8s.KubectlOptions { return s.kubectlOptions } +func (s *Scaffold) ControlAPIClient() (ControlAPIClient, error) { + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-control-api", 9090, 9090) + if err := tunnel.ForwardPortE(s.t); err != nil { + return nil, err + } + s.addFinalizers(tunnel.Close) + + return &controlAPI{ + client: NewClient("http", tunnel.Endpoint()), + }, nil +} + func NewClient(scheme, host string) *httpexpect.Expect { u := url.URL{ Scheme: scheme, @@ -443,3 +457,22 @@ func NewClient(scheme, host string) *httpexpect.Expect { ), }) } + +type ControlAPIClient interface { + ListServices() ([]any, int64, error) +} + +type controlAPI struct { + client *httpexpect.Expect +} + +func (c *controlAPI) ListServices() (result []any, total int64, err error) { + resp := c.client.Request(http.MethodGet, "/v1/services").Expect() + if resp.Raw().StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw()) + } + if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil { + return nil, 0, errors.Wrap(err, "failed to unmarshal response body") + } + return result, int64(len(result)), err +}