diff --git a/internal/manager/run.go b/internal/manager/run.go index 54ee31c8bb..b48746c162 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -21,7 +21,6 @@ import ( "context" "crypto/tls" "os" - "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -172,36 +171,6 @@ func Run(ctx context.Context, logger logr.Logger) error { return err } - go func() { - setupLog.Info("starting provider sync") - initalSyncDelay := config.ControllerConfig.ProviderConfig.InitSyncDelay.Duration - time.AfterFunc(initalSyncDelay, func() { - setupLog.Info("trying to initialize provider") - if err := provider.Sync(ctx); err != nil { - setupLog.Error(err, "unable to sync resources to provider") - return - } - }) - - syncPeriod := config.ControllerConfig.ProviderConfig.SyncPeriod.Duration - if syncPeriod < 1 { - return - } - ticker := time.NewTicker(syncPeriod) - defer ticker.Stop() - for { - select { - case <-ticker.C: - if err := provider.Sync(ctx); err != nil { - setupLog.Error(err, "unable to sync resources to provider") - return - } - case <-ctx.Done(): - return - } - } - }() - setupLog.Info("check ReferenceGrants is enabled") _, err = mgr.GetRESTMapper().KindsFor(schema.GroupVersionResource{ Group: v1beta1.GroupVersion.Group, diff --git a/test/e2e/framework/apisix_consts.go b/test/e2e/framework/apisix_consts.go index 42214982bb..a7ff0607c7 100644 --- a/test/e2e/framework/apisix_consts.go +++ b/test/e2e/framework/apisix_consts.go @@ -27,7 +27,8 @@ import ( ) var ( - ProviderType = cmp.Or(os.Getenv("PROVIDER_TYPE"), "apisix") + ProviderType = cmp.Or(os.Getenv("PROVIDER_TYPE"), "apisix") + ProviderSyncPeriod = cmp.Or(os.Getenv("PROVIDER_SYNC_PERIOD"), "200ms") ) var ( diff --git a/test/e2e/framework/manifests/apisix.yaml b/test/e2e/framework/manifests/apisix.yaml index affa4bfb57..568bde3df7 100644 --- a/test/e2e/framework/manifests/apisix.yaml +++ b/test/e2e/framework/manifests/apisix.yaml @@ -90,6 +90,9 @@ spec: - name: admin containerPort: 9180 protocol: TCP + - name: control + containerPort: 9090 + protocol: TCP volumeMounts: - name: config-writable mountPath: /usr/local/apisix/conf @@ -123,3 +126,18 @@ spec: selector: app.kubernetes.io/name: apisix type: {{ .ServiceType | default "NodePort" }} +--- +apiVersion: v1 +kind: Service +metadata: + name: apisix-control-api + labels: + app.kubernetes.io/name: apisix-control-api +spec: + ports: + - port: 9090 + name: control + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/name: apisix diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index c4bb101470..0bedfe142d 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -408,11 +408,11 @@ spec: periodSeconds: 10 resources: limits: - cpu: 500m - memory: 128Mi + cpu: "2" + memory: 1024Mi requests: - cpu: 10m - memory: 64Mi + cpu: "2" + memory: 1024Mi securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/test/e2e/load-test/e2e_test.go b/test/e2e/load-test/e2e_test.go new file mode 100644 index 0000000000..d28c30bf71 --- /dev/null +++ b/test/e2e/load-test/e2e_test.go @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package load_test + +import ( + "fmt" + "io" + "os" + "testing" + "time" + + "github.com/api7/gopkg/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var closer io.Closer + +func init() { + // save log locally + file, err := os.Create(time.Now().Format("load_test_200601021504.log")) + if err != nil { + log.Fatalf("failed to create log file, err: %v", err) + } + closer = file + GinkgoWriter.TeeTo(file) +} + +// Run long-term-stability tests using Ginkgo runner. +func TestLongTermStability(t *testing.T) { + defer func() { _ = closer.Close() }() + + RegisterFailHandler(Fail) + var f = framework.NewFramework() + _ = f + + scaffold.NewDeployer = func(s *scaffold.Scaffold) scaffold.Deployer { + return scaffold.NewAPISIXDeployer(s) + } + + _, _ = fmt.Fprintf(GinkgoWriter, "Starting load-test suite\n") + RunSpecs(t, "long-term-stability suite") +} diff --git a/test/e2e/load-test/spec_subject.go b/test/e2e/load-test/spec_subject.go new file mode 100644 index 0000000000..79de2cf5f3 --- /dev/null +++ b/test/e2e/load-test/spec_subject.go @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package load + +import ( + "bytes" + "context" + "fmt" + "net/http" + "time" + + "github.com/api7/gopkg/pkg/log" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/apache/apisix-ingress-controller/test/e2e/framework" + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +const gatewayProxyYaml = ` +apiVersion: apisix.apache.org/v1alpha1 +kind: GatewayProxy +metadata: + name: apisix-proxy-config +spec: + provider: + type: ControlPlane + controlPlane: + service: + name: %s + port: 9180 + auth: + type: AdminKey + adminKey: + value: "%s" +` + +const ingressClassYaml = ` +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + name: apisix +spec: + controller: "apisix.apache.org/apisix-ingress-controller" + parameters: + apiGroup: "apisix.apache.org" + kind: "GatewayProxy" + name: "apisix-proxy-config" + namespace: %s + scope: "Namespace" +` + +var _ = Describe("Load Test", func() { + var ( + s = scaffold.NewScaffold(&scaffold.Options{ + ControllerName: "apisix.apache.org/apisix-ingress-controller", + }) + controlAPIClient scaffold.ControlAPIClient + err error + ) + + BeforeEach(func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, framework.ProviderType, s.AdminKey()) + err = s.CreateResourceFromStringWithNamespace(gatewayProxy, s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create IngressClass") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingressClassYaml, s.Namespace()), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass") + time.Sleep(5 * time.Second) + + By("port-forward to control api service") + controlAPIClient, err = s.ControlAPIClient() + Expect(err).NotTo(HaveOccurred(), "create control api client") + }) + + Context("Load Test 2000 ApisixRoute", func() { + It("test 2000 ApisixRoute", func() { + const total = 2000 + + const apisixRouteSpec = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /get + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + + By(fmt.Sprintf("prepare %d ApisixRoutes", total)) + var text = bytes.NewBuffer(nil) + for i := range total { + name := getRouteName(i) + _, err := fmt.Fprintf(text, apisixRouteSpec, name, name) + Expect(err).NotTo(HaveOccurred()) + text.WriteString("\n---\n") + } + err := s.CreateResourceFromString(text.String()) + Expect(err).NotTo(HaveOccurred(), "creating ApisixRoutes") + + var ( + results []TestResult + now = time.Now() + ) + By("Test the time required for applying a large number of ApisixRoutes to take effect") + var times int + err = wait.PollUntilContextTimeout(context.Background(), time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { + times++ + results, _, err := controlAPIClient.ListServices() + if err != nil { + log.Errorw("failed to ListServices", zap.Error(err)) + return false, nil + } + if len(results) != total { + log.Debugw("number of effective services", zap.Int("number", len(results)), zap.Int("times", times)) + return false, nil + } + return len(results) == total, nil + }) + Expect(err).ShouldNot(HaveOccurred()) + results = append(results, TestResult{ + CaseName: fmt.Sprintf("Apply %d ApisixRoutes", total), + CostTime: time.Since(now), + }) + + By("Test the time required for an ApisixRoute update to take effect") + var apisixRouteSpec0 = ` +apiVersion: apisix.apache.org/v2 +kind: ApisixRoute +metadata: + name: %s +spec: + ingressClassName: apisix + http: + - name: rule0 + match: + paths: + - /headers + exprs: + - subject: + scope: Header + name: X-Route-Name + op: Equal + value: %s + backends: + - serviceName: httpbin-service-e2e-test + servicePort: 80 +` + name := getRouteName(10) + err = s.CreateResourceFromString(fmt.Sprintf(apisixRouteSpec0, name, name)) + Expect(err).NotTo(HaveOccurred()) + now = time.Now() + Eventually(func() int { + return s.NewAPISIXClient().GET("/headers").WithHeader("X-Route-Name", name).Expect().Raw().StatusCode + }).WithTimeout(time.Minute).ProbeEvery(100 * time.Millisecond).Should(Equal(http.StatusOK)) + results = append(results, TestResult{ + CaseName: fmt.Sprintf("Update a single ApisixRoute base on %d ApisixRoutes", total), + CostTime: time.Since(now), + }) + + PrintResults(results) + }) + }) +}) + +func getRouteName(i int) string { + return fmt.Sprintf("test-route-%04d", i) +} + +type TestResult struct { + CaseName string + CostTime time.Duration +} + +func (tr TestResult) String() string { + return fmt.Sprintf("%s takes effect for %s", tr.CaseName, tr.CostTime) +} + +func PrintResults(results []TestResult) { + fmt.Printf("\n======================TEST RESULT ProviderSyncPeriod %s===============================\n", framework.ProviderSyncPeriod) + fmt.Printf("%-70s", "Test Case") + fmt.Printf("%-70s\n", "Time Required") + fmt.Printf("%-70s\n", "--------------------------------------------------------------------------------------") + for _, result := range results { + fmt.Printf("%-70s", result.CaseName) + fmt.Printf("%-70s\n", result.CostTime) + } + fmt.Println("======================================================================================") + fmt.Println() +} diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index b6fe7febcf..b5240cd9c0 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -260,7 +260,7 @@ func (s *APISIXDeployer) DeployIngress() { s.Framework.DeployIngress(framework.IngressDeployOpts{ ControllerName: s.opts.ControllerName, ProviderType: framework.ProviderType, - ProviderSyncPeriod: 200 * time.Millisecond, + ProviderSyncPeriod: getProviderSyncPeriod(), Namespace: s.namespace, Replicas: 1, }) @@ -270,12 +270,20 @@ func (s *APISIXDeployer) ScaleIngress(replicas int) { s.Framework.DeployIngress(framework.IngressDeployOpts{ ControllerName: s.opts.ControllerName, ProviderType: framework.ProviderType, - ProviderSyncPeriod: 200 * time.Millisecond, + ProviderSyncPeriod: getProviderSyncPeriod(), Namespace: s.namespace, Replicas: replicas, }) } +func getProviderSyncPeriod() time.Duration { + providerSyncPeriod, err := time.ParseDuration(framework.ProviderSyncPeriod) + if err != nil { + providerSyncPeriod = 200 * time.Millisecond + } + return providerSyncPeriod +} + // getEnvOrDefault returns environment variable value or default func getEnvOrDefault(key, defaultValue string) string { if value := os.Getenv(key); value != "" { diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index fe533a82cf..790a33eddc 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -20,6 +20,7 @@ package scaffold import ( "context" "crypto/tls" + "encoding/json" "fmt" "net/http" "net/url" @@ -31,6 +32,7 @@ import ( "github.com/gruntwork-io/terratest/modules/testing" . "github.com/onsi/ginkgo/v2" //nolint:staticcheck . "github.com/onsi/gomega" //nolint:staticcheck + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -425,6 +427,18 @@ func (s *Scaffold) KubeOpts() *k8s.KubectlOptions { return s.kubectlOptions } +func (s *Scaffold) ControlAPIClient() (ControlAPIClient, error) { + tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-control-api", 9090, 9090) + if err := tunnel.ForwardPortE(s.t); err != nil { + return nil, err + } + s.addFinalizers(tunnel.Close) + + return &controlAPI{ + client: NewClient("http", tunnel.Endpoint()), + }, nil +} + func NewClient(scheme, host string) *httpexpect.Expect { u := url.URL{ Scheme: scheme, @@ -443,3 +457,22 @@ func NewClient(scheme, host string) *httpexpect.Expect { ), }) } + +type ControlAPIClient interface { + ListServices() ([]any, int64, error) +} + +type controlAPI struct { + client *httpexpect.Expect +} + +func (c *controlAPI) ListServices() (result []any, total int64, err error) { + resp := c.client.Request(http.MethodGet, "/v1/services").Expect() + if resp.Raw().StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw()) + } + if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil { + return nil, 0, errors.Wrap(err, "failed to unmarshal response body") + } + return result, int64(len(result)), err +}