large scale HTTPRoute

Author: dspo

test/e2e/scaffold/k8s.go

@@ -110,6 +110,14 @@ func (s *Scaffold) ListPodsByLabels(labels string) ([]corev1.Pod, error) {
 	})
 }
 
+func (s *Scaffold) DeleteResourcesByLabels(resourceType, label string) error {
+	return k8s.RunKubectlE(s.t, s.kubectlOptions, "delete", resourceType, "-l", label)
+}
+
+func (s *Scaffold) GetResourcesByLabelsOutput(resourceType, label string) (string, error) {
+	return k8s.RunKubectlAndGetOutputE(s.t, s.kubectlOptions, "get", resourceType, "-l", label)
+}
+
 // CreateResourceFromStringWithNamespace creates resource from a loaded yaml string
 // and sets its namespace to the specified one.
 func (s *Scaffold) CreateResourceFromStringWithNamespace(yaml, namespace string) error {

test/e2e/scaffold/locust.go

@@ -25,8 +25,21 @@ data:
 
     class HttpbinRequester(HttpUser):
         @task 
-        def request_headers(self): 
+        def headers(self): 
             self.client.get("/headers", headers={"Host": "httpbin.example"})
+
+        @task
+        def get(self):
+            self.client.get("/get", headers={"Host": "httpbin.example"})
+
+        @task
+        def post(self):
+            self.client.post("/post", headers={"Host": "httpbin.example"})
+
+        @task
+        def image(self):
+            self.client.image("/image", headers={"Host": "httpbin.example"})
+
   LOCUST_HOST: http://api7ee3-apisix-gateway-mtls:9080
   LOCUST_SPAWN_RATE: "50"
   LOCUST_USERS: "500"
@@ -116,25 +129,6 @@ func (s *Scaffold) DeployLocust() *corev1.Service {
 	return service
 }
 
-// func (s *Scaffold) LocustClient() *httpexpect.Expect {
-// 	u := url.URL{
-// 		Scheme: "http",
-// 		Host: s.locustTunnel.Endpoint(),
-// 	}
-// 	return httpexpect.WithConfig(httpexpect.Config{
-// 		BaseURL: u.String(),
-// 		Client: &http.Client{
-// 			Transport: &http.Transport{},
-// 			CheckRedirect: func(req *http.Request, via []*http.Request) error {
-// 				return http.ErrUseLastResponse
-// 			},
-// 		},
-// 		Reporter: httpexpect.NewAssertReporter(
-// 			httpexpect.NewAssertReporter(s.GinkgoT),
-// 		),
-// 	})
-// }
-
 func (s *Scaffold) ResetLocust() error {
 	if s.locustTunnel == nil {
 		return errors.New("locust is not deployed")
@@ -143,7 +137,9 @@ func (s *Scaffold) ResetLocust() error {
 	if err != nil {
 		return errors.Wrap(err, "failed to request reset locust")
 	}
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	if resp.StatusCode != http.StatusOK {
 		return errors.Errorf("request reset locust not OK, status: %s", resp.Status)
 	}
@@ -162,7 +158,9 @@ func (s *Scaffold) DownloadLocustReport(filename string) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to request download report")
 	}
-	defer resp.Body.Close()
+	defer func() {
+		_ = resp.Body.Close()
+	}()
 	if resp.StatusCode != http.StatusOK {
 		return errors.Errorf("request download report not OK, status: %s", resp.Status)
 	}

test/e2e/scaffold/scaffold.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/gavv/httpexpect/v2"
 	"github.com/gruntwork-io/terratest/modules/k8s"
+	"github.com/gruntwork-io/terratest/modules/logger"
 	"github.com/gruntwork-io/terratest/modules/testing"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -74,6 +75,8 @@ type Options struct {
 
 	GinkgoBeforeCallback func(...any) bool
 	GinkgoAfterCallback  func(...any) bool
+
+	KubectlLogger *logger.Logger
 }
 
 type Scaffold struct {
@@ -365,6 +368,9 @@ func (s *Scaffold) beforeEach() {
 		ConfigPath: s.opts.Kubeconfig,
 		Namespace:  s.namespace,
 	}
+	if s.opts.KubectlLogger != nil {
+		s.kubectlOptions.Logger = s.opts.KubectlLogger
+	}
 	if s.opts.ControllerName == "" {
 		s.opts.ControllerName = fmt.Sprintf("%s/%d", DefaultControllerName, time.Now().Nanosecond())
 	}

test/long_term_stability/spec_subjects/spec_subjects.go

@@ -3,11 +3,16 @@ package spec_subjects
 import (
 	"fmt"
 	"net/http"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/api7/api7-ingress-controller/test/e2e/scaffold"
+	"github.com/gruntwork-io/terratest/modules/logger"
+	"github.com/gruntwork-io/terratest/modules/retry"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
 )
 
 var _ = Describe("API7 Ingress Controller Long Term Stability Tests", Ordered, func() {
@@ -16,6 +21,7 @@ var _ = Describe("API7 Ingress Controller Long Term Stability Tests", Ordered, f
 			ControllerName:       "gateway.api7.io/api7-ingress-controller",
 			GinkgoBeforeCallback: BeforeAll,
 			GinkgoAfterCallback:  AfterAll,
+			KubectlLogger:        logger.Discard, // too many logs in long-term stability test so discard kubectl apply logs
 		})
 		it int
 	)
@@ -55,6 +61,37 @@ spec:
     - path:
         type: Exact
         value: /headers
+    - path:
+        type: Exact
+        value: /get
+    - path:
+        type: Exact
+        value: /post
+    - path:
+        type: Exact
+        value: /image
+    
+    backendRefs:
+    - name: httpbin-service-e2e-test
+      port: 80
+`
+		httpRouteTemplate2 = `
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: %s
+  labels:
+    template_name: httpRouteTemplate2
+spec:
+  parentRefs:
+  - name: api7ee
+  hostnames:
+  - httpbin.example
+  rules:
+  - matches:
+    - path:
+        type: Exact
+        value: %s
     backendRefs:
     - name: httpbin-service-e2e-test
       port: 80
@@ -124,12 +161,12 @@ spec:
 
 	Context("Service Discovery", func() {
 		BeforeEach(func() {
-			// scale backend pods replicas
+			By("scale backend pods replicas")
 			err := s.ScaleHTTPBIN(5)
 			Expect(err).NotTo(HaveOccurred(), "scaling httpbin")
 		})
 
-		It("service discovery", func() {
+		It("rolling update", func() {
 			var total = 5
 			for i := 0; i < total; i++ {
 				By(fmt.Sprintf("rolling update deployment/httpbin-deployment-e2e-test [%02d/%02d]", i+1, total))
@@ -139,32 +176,120 @@ spec:
 				time.Sleep(time.Minute)
 			}
 
-			err := s.DownloadLocustReport(fmt.Sprintf("%02d_service_discovery", it))
+			err := s.DownloadLocustReport(fmt.Sprintf("%02d_rolling_update", it))
+			Expect(err).NotTo(HaveOccurred(), "getting locust report")
+		})
+
+		It("scale replicas", func() {
+			var total = 5
+			for i := 0; i < total; i++ {
+				By(fmt.Sprintf("scale replicas for deployment/httpbin-deployment-e2e-test [%02d/%02d]", i+1, total))
+				err := s.ScaleHTTPBIN(4 + i%2*3) // scale to 4 if "i" is even, scale to 7 if "i" is odd.
+				Expect(err).NotTo(HaveOccurred(), "scale replicas for deployment/httpbin-deployment-e2e-test")
+				time.Sleep(time.Minute)
+			}
+
+			err := s.DownloadLocustReport(fmt.Sprintf("%02d_scale_replicas", it))
 			Expect(err).NotTo(HaveOccurred(), "getting locust report")
 		})
 	})
 
 	Context("Large-scale HTTPRoute", func() {
-		It("it 0", func() {
-			Ω(true).Should(BeTrue())
-		})
+		var (
+			reconcileDurationPerHTTPRoute = 3 * time.Second
+			probeDuration                 = 100 * time.Millisecond
+
+			resourceTypeHTTPRoute = "HTTPRoute"
+			label                 = "template_name=httpRouteTemplate2"
+		)
+
+		for _, total := range []int{500, 2000, 5000} {
+			var (
+				reconcileDurationBatchProcess = time.Duration(total) * reconcileDurationPerHTTPRoute
+				title                         = strconv.FormatInt(int64(total), 10) + " HTTPRoute"
+			)
+
+			It(title, func() {
+				defer func() {
+					By("cleaning up HTTPRoutes")
+					err := s.DeleteResourcesByLabels(resourceTypeHTTPRoute, label)
+					Expect(err).NotTo(HaveOccurred(), "delete HTTPRoute by label")
+
+					Eventually(func() string {
+						output, err := s.GetResourcesByLabelsOutput(resourceTypeHTTPRoute, label)
+						Expect(err).NotTo(HaveOccurred(), "getting HTTPRoute")
+						return output
+					}).WithTimeout(reconcileDurationBatchProcess).ProbeEvery(probeDuration).
+						Should(ContainSubstring("No resources found"))
+				}()
+
+				By("prepare HTTPRoutes")
+				for i := 0; i < total+100; i++ {
+					By(fmt.Sprintf("prepare HTTPRoutes [%04d/%04d]", i+1, total))
+					routeName := "httpbin-" + strconv.FormatInt(int64(i), 10)
+					pathValue := "/delay/" + strconv.FormatInt(int64(i), 10)
+					err := s.CreateResourceFromString(fmt.Sprintf(httpRouteTemplate2, routeName, pathValue))
+					Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute")
+
+					message := retry.DoWithRetry(s.GinkgoT, "Wait for HTTPRoute ok", 100, time.Second, func() (string, error) {
+						yaml_, err := s.GetResourceYaml(resourceTypeHTTPRoute, routeName)
+						if err != nil {
+							return "", err
+						}
+						if !strings.Contains(yaml_, `status: "True"`) {
+							return "", errors.New("HTTPRoute status is not True")
+						}
+						return "HTTPRoute is now available", nil
+					},
+					)
+					s.Logf(message)
+				}
+
+				By("delete 100 HTTPRoutes")
+				for i := total; i < total+100; i++ {
+					By(fmt.Sprintf("prepare 1000 HTTPRoute [%04d/%04d]", i, total+100))
+					routeName := "httpbin-" + strconv.FormatInt(int64(i), 10)
+					err := s.DeleteResource(resourceTypeHTTPRoute, routeName)
+					Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute")
+
+					Eventually(func() string {
+						_, err := s.GetResourceYaml(resourceTypeHTTPRoute, "")
+						return err.Error()
+					}).WithTimeout(reconcileDurationPerHTTPRoute).ProbeEvery(probeDuration).
+						Should(ContainSubstring("not found"))
+				}
+
+				err := s.DownloadLocustReport(fmt.Sprintf("%02d_large_scale_httproute(1000)", it))
+				Expect(err).NotTo(HaveOccurred(), "getting locust report")
+			})
+		}
 	})
 
-	Context("Ingress Controller is crashing", func() {
+	PContext("Ingress Controller is crashing", func() {
 		It("it 0", func() {
 			Ω(true).Should(BeTrue())
 		})
 	})
 
-	Context("Regression Tests", func() {
-		It("Under large-scale HTTPRoute, some HTTPRoute resources are abnormal, which cannot affect the synchronization efficient of other resources", func() {})
+	PContext("Regression Tests", func() {
+		// Under large-scale HTTPRoute, some HTTPRoute resources are abnormal, which cannot affect the synchronization
+		// efficient of other resources
+		It("", func() {})
 
-		It("Under large-scale CRDs, some CRDs resources are abnormal, which cannot affect the synchronization efficient of other resources", func() {})
+		// Under large-scale CRDs, some CRDs resources are abnormal, which cannot affect the synchronization efficient
+		// of other resources
+		It("", func() {})
 
-		It("When a large number of CRDs are applied concurrently, the processing capacity of IngressController is linear (≤O(n), n is the number of resources applied simultaneously)", func() {})
+		// When a large number of CRDs are applied concurrently, the processing capacity of IngressController is linear
+		// (≤O(n), n is the number of resources applied simultaneously)
+		It("", func() {})
 
-		It("Under large-scale CRDs, some CRDs are added/deleted/modified concurrently, and the processing capacity of IngressController has no obvious relationship with the number of existing CRDs", func() {})
+		// Under large-scale CRDs, some CRDs are added/deleted/modified concurrently, and the processing capacity of
+		// IngressController has no obvious relationship with the number of existing CRDs
+		It("", func() {})
 
-		It("When IngressController is unexpectedly unavailable, it does not affect the existing configuration of the data plane", func() {})
+		// When IngressController is unexpectedly unavailable, it does not affect the existing configuration of the
+		// data plane
+		It("", func() {})
 	})
 })