From dd3a521ee078cc93738997c53516cd5d766f63f5 Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Tue, 2 Sep 2025 10:43:28 -0700 Subject: [PATCH 1/8] add fixes for scale and zero downtime scale tests --- tests/suite/manifests/scale/httproute.yaml | 18 ++++++ tests/suite/manifests/scale/upstreams.yaml | 19 ------ tests/suite/scale_test.go | 71 ++++++++++------------ 3 files changed, 50 insertions(+), 58 deletions(-) create mode 100644 tests/suite/manifests/scale/httproute.yaml diff --git a/tests/suite/manifests/scale/httproute.yaml b/tests/suite/manifests/scale/httproute.yaml new file mode 100644 index 0000000000..51ecb72cb6 --- /dev/null +++ b/tests/suite/manifests/scale/httproute.yaml @@ -0,0 +1,18 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route +spec: + parentRefs: + - name: gateway + sectionName: listener + hostnames: + - "*.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: backend + port: 80 diff --git a/tests/suite/manifests/scale/upstreams.yaml b/tests/suite/manifests/scale/upstreams.yaml index 1c9587f31c..0626024f93 100644 --- a/tests/suite/manifests/scale/upstreams.yaml +++ b/tests/suite/manifests/scale/upstreams.yaml @@ -13,25 +13,6 @@ spec: port: 80 protocol: HTTP --- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: route -spec: - parentRefs: - - name: gateway - sectionName: listener - hostnames: - - "*.example.com" - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - name: backend - port: 80 ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 482740b726..1f3950bd27 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -40,6 +40,10 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { "scale/upstreams.yaml", } + httpRouteManifests = []string{ + "scale/httproute.yaml", + } + namespace = "scale" scrapeInterval = 15 * time.Second @@ -468,6 +472,10 @@ The logs are attached only if there are errors. Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + // apply HTTPRoute after upstreams are ready + Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + var nginxPodNames []string var err error Eventually( @@ -835,7 +843,19 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim AfterAll(func() { _, err := fmt.Fprint(outFile) Expect(err).ToNot(HaveOccurred()) - Expect(outFile.Close()).To(Succeed()) + + // check if file is already closed or not + if outFile != nil { + err = outFile.Close() + if err != nil { + // warning only + if strings.Contains(err.Error(), "file already closed") { + GinkgoWriter.Printf("Warning: attempted to close already closed file: %v\n", err) + } else { + Expect(err).ToNot(HaveOccurred()) + } + } + } // restoring NGF shared among tests in the suite cfg := getDefaultSetupCfg() @@ -1012,59 +1032,32 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim checkGatewayListeners := func(num int) { Eventually( func() error { - ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout) + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout*2) defer cancel() var gw v1.Gateway key := types.NamespacedName{Namespace: ns.Name, Name: "gateway"} if err := resourceManager.K8sClient.Get(ctx, key, &gw); err != nil { - return err + return fmt.Errorf("failed to get gateway: %w", err) } - if len(gw.Status.Listeners) != num { - return fmt.Errorf("gateway listeners not updated to %d entries", num) + currentListeners := len(gw.Status.Listeners) + currentGen := gw.Status + specGen := gw.Generation + + if currentListeners != num { + return fmt.Errorf("gateway listeners: got %d, want %d (observedGen: %v, specGen: %v)", + currentListeners, num, currentGen, specGen) } return nil }, ). - WithTimeout(5 * time.Second). - WithPolling(100 * time.Millisecond). + WithTimeout(timeoutConfig.RequestTimeout). + WithPolling(1 * time.Second). Should(Succeed()) } - It("scales up abruptly without downtime", func() { - _, err := fmt.Fprint(outFile, "\n### Scale Up Abruptly\n") - Expect(err).ToNot(HaveOccurred()) - - testFileNamePrefix := formatTestFileNamePrefix("abrupt-scale-up", test.valuesFile) - - var wg sync.WaitGroup - for _, test := range trafficConfigs { - wg.Add(1) - go func(cfg trafficCfg) { - defer GinkgoRecover() - defer wg.Done() - - sendTraffic(cfg, testFileNamePrefix, 2*time.Minute) - }(test) - } - - // allow traffic flow to start - time.Sleep(2 * time.Second) - - Expect(resourceManager.ScaleNginxDeployment(ngfNamespace, releaseName, int32(test.numReplicas))).To(Succeed()) - Expect(resourceManager.ApplyFromFiles([]string{"scale/zero-downtime/gateway-2.yaml"}, ns.Name)).To(Succeed()) - checkGatewayListeners(3) - - wg.Wait() - close(metricsCh) - - for res := range metricsCh { - writeResults(testFileNamePrefix, res) - } - }) - It("scales down abruptly without downtime", func() { _, err := fmt.Fprint(outFile, "\n### Scale Down Abruptly\n") Expect(err).ToNot(HaveOccurred()) From 541b38c36a2327e625c4cf2c325261bae9b21693 Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Tue, 2 Sep 2025 10:45:11 -0700 Subject: [PATCH 2/8] update gateway API CRD installation with --server-side flag --- Makefile | 2 +- tests/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index cc408afc54..c794b3a9e2 100644 --- a/Makefile +++ b/Makefile @@ -130,7 +130,7 @@ install-crds: ## Install CRDs .PHONY: install-gateway-crds install-gateway-crds: ## Install Gateway API CRDs - kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply -f - + kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply --server-side -f - .PHONY: uninstall-gateway-crds uninstall-gateway-crds: ## Uninstall Gateway API CRDs diff --git a/tests/Makefile b/tests/Makefile index f1ccfdd3c5..3a0635fba6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -176,7 +176,7 @@ HELM_PARAMETERS += --set nginxGateway.name=nginx-gateway --set nginx.service.typ # it overrides the target in the main Makefile when the GW_API_VERSION is set to main ifeq ($(GW_API_VERSION),main) install-gateway-crds: - kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply -f - + kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply --server-side -f - endif .PHONY: install-ngf-local-no-build From b5349e6e99616b67e65f19ef615694090a4b9b1b Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Tue, 2 Sep 2025 12:28:32 -0700 Subject: [PATCH 3/8] add zero downtime scale tests --- tests/suite/scale_test.go | 41 ++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 1f3950bd27..ef6e11887c 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -1041,13 +1041,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim return fmt.Errorf("failed to get gateway: %w", err) } - currentListeners := len(gw.Status.Listeners) - currentGen := gw.Status - specGen := gw.Generation - - if currentListeners != num { - return fmt.Errorf("gateway listeners: got %d, want %d (observedGen: %v, specGen: %v)", - currentListeners, num, currentGen, specGen) + if len(gw.Status.Listeners) != num { + return fmt.Errorf("gateway listeners not updated to %d entries", num) } return nil @@ -1058,6 +1053,38 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim Should(Succeed()) } + It("scales up abruptly without downtime", func() { + _, err := fmt.Fprint(outFile, "\n### Scale Up Abruptly\n") + Expect(err).ToNot(HaveOccurred()) + + testFileNamePrefix := formatTestFileNamePrefix("abrupt-scale-up", test.valuesFile) + + var wg sync.WaitGroup + for _, test := range trafficConfigs { + wg.Add(1) + go func(cfg trafficCfg) { + defer GinkgoRecover() + defer wg.Done() + + sendTraffic(cfg, testFileNamePrefix, 2*time.Minute) + }(test) + } + + // allow traffic flow to start + time.Sleep(2 * time.Second) + + Expect(resourceManager.ScaleNginxDeployment(ngfNamespace, releaseName, int32(test.numReplicas))).To(Succeed()) + Expect(resourceManager.ApplyFromFiles([]string{"scale/zero-downtime/gateway-2.yaml"}, ns.Name)).To(Succeed()) + checkGatewayListeners(3) + + wg.Wait() + close(metricsCh) + + for res := range metricsCh { + writeResults(testFileNamePrefix, res) + } + }) + It("scales down abruptly without downtime", func() { _, err := fmt.Fprint(outFile, "\n### Scale Down Abruptly\n") Expect(err).ToNot(HaveOccurred()) From 5badd37200fce32afb4f523fc93c7008989eee35 Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:55:27 -0700 Subject: [PATCH 4/8] update file closing logic --- tests/suite/scale_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index ef6e11887c..e132b66630 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -844,12 +845,9 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim _, err := fmt.Fprint(outFile) Expect(err).ToNot(HaveOccurred()) - // check if file is already closed or not if outFile != nil { - err = outFile.Close() - if err != nil { - // warning only - if strings.Contains(err.Error(), "file already closed") { + if err := outFile.Close(); err != nil { + if errors.Is(err, os.ErrClosed) || strings.Contains(err.Error(), "file already closed") { GinkgoWriter.Printf("Warning: attempted to close already closed file: %v\n", err) } else { Expect(err).ToNot(HaveOccurred()) @@ -1048,7 +1046,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim return nil }, ). - WithTimeout(timeoutConfig.RequestTimeout). + WithTimeout(timeoutConfig.GetStatusTimeout). WithPolling(1 * time.Second). Should(Succeed()) } From f9a4ca4162e1c77d00661bf418ca67ff266129c9 Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Wed, 3 Sep 2025 14:34:27 -0600 Subject: [PATCH 5/8] add timeout for updating gateway listeners --- tests/framework/timeout.go | 28 ++++++++++++++++------------ tests/suite/scale_test.go | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/tests/framework/timeout.go b/tests/framework/timeout.go index 394f9e351d..7c47b191bd 100644 --- a/tests/framework/timeout.go +++ b/tests/framework/timeout.go @@ -38,22 +38,26 @@ type TimeoutConfig struct { // KubernetesClientTimeout represents the maximum time for Kubernetes client operations. KubernetesClientTimeout time.Duration + + // GatewayListenerUpdateTimeout represents the maximum time for Gateway Listener count to be updated. + GatewayListenerUpdateTimeout time.Duration } // DefaultTimeoutConfig populates a TimeoutConfig with the default values. func DefaultTimeoutConfig() TimeoutConfig { return TimeoutConfig{ - CreateTimeout: 60 * time.Second, - UpdateTimeout: 60 * time.Second, - DeleteTimeout: 10 * time.Second, - DeleteNamespaceTimeout: 150 * time.Second, - GetTimeout: 10 * time.Second, - ManifestFetchTimeout: 10 * time.Second, - RequestTimeout: 10 * time.Second, - ContainerRestartTimeout: 10 * time.Second, - GetLeaderLeaseTimeout: 60 * time.Second, - GetStatusTimeout: 60 * time.Second, - TestForTrafficTimeout: 60 * time.Second, - KubernetesClientTimeout: 10 * time.Second, + CreateTimeout: 60 * time.Second, + UpdateTimeout: 60 * time.Second, + DeleteTimeout: 10 * time.Second, + DeleteNamespaceTimeout: 150 * time.Second, + GetTimeout: 10 * time.Second, + ManifestFetchTimeout: 10 * time.Second, + RequestTimeout: 30 * time.Second, + ContainerRestartTimeout: 10 * time.Second, + GetLeaderLeaseTimeout: 60 * time.Second, + GetStatusTimeout: 60 * time.Second, + TestForTrafficTimeout: 60 * time.Second, + KubernetesClientTimeout: 10 * time.Second, + GatewayListenerUpdateTimeout: 60 * time.Second, } } diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index e132b66630..fba6430470 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -1046,7 +1046,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim return nil }, ). - WithTimeout(timeoutConfig.GetStatusTimeout). + WithTimeout(timeoutConfig.GatewayListenerUpdateTimeout). WithPolling(1 * time.Second). Should(Succeed()) } From e9109815450c0dc35181a9a2fbd5f93792eaba0b Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Thu, 4 Sep 2025 16:20:39 -0600 Subject: [PATCH 6/8] make NFR test outputs more verbose --- tests/Makefile | 2 +- tests/framework/generate_manifests.go | 109 +++++++-- tests/suite/scale_test.go | 330 +++++++++++++------------- 3 files changed, 250 insertions(+), 191 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 3a0635fba6..668fd879e3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -122,7 +122,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) CGO_ENABLED=1 go run github.com/onsi/ginkgo/v2/ginkgo --race --randomize-all --randomize-suites --keep-going --fail-on-pending \ - --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) --flake-attempts=2 \ + --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) --flake-attempts=1 --show-node-events --output-interceptor-mode=none \ --label-filter "nfr" $(GINKGO_FLAGS) --timeout 5h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ --gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ diff --git a/tests/framework/generate_manifests.go b/tests/framework/generate_manifests.go index e0ad091585..e4a2d05a9a 100644 --- a/tests/framework/generate_manifests.go +++ b/tests/framework/generate_manifests.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log" "text/template" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -150,13 +151,17 @@ func decodeObjects(reader io.Reader) ([]client.Object, error) { } // GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test. +// Modified to create all services and gateways first, then scale routes. func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) { + log.Printf("GenerateScaleListenerObjects: Starting generation for %d listeners, TLS=%v", numListeners, tls) var result ScaleObjects listeners := make([]listener, 0) backends := make([]string, 0) secrets := make([]string, 0) + routes := make([]route, 0) + // Generate all listeners, backends, and routes for i := range numListeners { listenerName := fmt.Sprintf("listener-%d", i) hostnamePrefix := fmt.Sprintf("%d", i) @@ -181,39 +186,65 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err BackendName: backendName, } + routes = append(routes, r) backends = append(backends, backendName) + } - objects, err := generateManifests(listeners, []route{r}) - if err != nil { - return ScaleObjects{}, err - } + log.Printf("GenerateScaleListenerObjects: Generated %d listeners, %d backends, %d secrets", len(listeners), len(backends), len(secrets)) - result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects) + // Generate gateway with all listeners + log.Printf("GenerateScaleListenerObjects: Generating gateway with %d listeners", len(listeners)) + gatewayObjects, err := generateManifests(listeners, []route{}) + if err != nil { + return ScaleObjects{}, err } + log.Printf("GenerateScaleListenerObjects: Generated %d gateway objects", len(gatewayObjects)) + // Generate secrets if TLS is enabled secretObjects, err := generateSecrets(secrets) if err != nil { return ScaleObjects{}, err } + log.Printf("GenerateScaleListenerObjects: Generated %d secret objects", len(secretObjects)) - result.BaseObjects = append(result.BaseObjects, secretObjects...) - + // Generate backend app objects (services, deployments) + log.Printf("GenerateScaleListenerObjects: Generating backend objects for %d backends", len(backends)) backendObjects, err := generateBackendAppObjects(backends) if err != nil { return ScaleObjects{}, err } + log.Printf("GenerateScaleListenerObjects: Generated %d backend objects", len(backendObjects)) + // BaseObjects includes services, secrets, and gateway with all listeners + result.BaseObjects = append(result.BaseObjects, secretObjects...) result.BaseObjects = append(result.BaseObjects, backendObjects...) + result.BaseObjects = append(result.BaseObjects, gatewayObjects...) + log.Printf("GenerateScaleListenerObjects: BaseObjects contains %d total objects", len(result.BaseObjects)) + // Each iteration creates one route + for i := range numListeners { + log.Printf("GenerateScaleListenerObjects: Generating route %d: %s", i, routes[i].Name) + objects, err := generateManifests([]listener{}, []route{routes[i]}) + if err != nil { + return ScaleObjects{}, err + } + + result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects) + log.Printf("GenerateScaleListenerObjects: Route %d generated %d objects", i, len(objects)) + } + + log.Printf("GenerateScaleListenerObjects: Completed. BaseObjects: %d, ScaleIterationGroups: %d", len(result.BaseObjects), len(result.ScaleIterationGroups)) return result, nil } func generateSecrets(secrets []string) ([]client.Object, error) { + log.Printf("generateSecrets: Generating %d secrets: %v", len(secrets), secrets) objects := make([]client.Object, 0, len(secrets)) - for _, secret := range secrets { + for i, secret := range secrets { var buf bytes.Buffer + log.Printf("generateSecrets: Executing secret template for secret %d: %s", i, secret) if err := secretTmpl.Execute(&buf, secret); err != nil { return nil, err } @@ -223,14 +254,18 @@ func generateSecrets(secrets []string) ([]client.Object, error) { return nil, err } + log.Printf("generateSecrets: Secret %s generated %d objects", secret, len(objs)) objects = append(objects, objs...) } + log.Printf("generateSecrets: Total secret objects generated: %d", len(objects)) return objects, nil } // GenerateScaleHTTPRouteObjects generates objects for a given number of routes for the scale test. +// Modified to create all services and gateways first, then scale routes. func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { + log.Printf("GenerateScaleHTTPRouteObjects: Starting generation for %d routes", numRoutes) var result ScaleObjects l := listener{ @@ -240,6 +275,27 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { backendName := "backend" + // Generate the Gateway once and add it to BaseObjects + log.Printf("GenerateScaleHTTPRouteObjects: Generating gateway with single listener") + gatewayObjects, err := generateManifests([]listener{l}, []route{}) + if err != nil { + return ScaleObjects{}, err + } + log.Printf("GenerateScaleHTTPRouteObjects: Generated %d gateway objects", len(gatewayObjects)) + + // Generate backend app objects (services, deployments) + log.Printf("GenerateScaleHTTPRouteObjects: Generating backend objects for backend: %s", backendName) + backendObjects, err := generateBackendAppObjects([]string{backendName}) + if err != nil { + return ScaleObjects{}, err + } + log.Printf("GenerateScaleHTTPRouteObjects: Generated %d backend objects", len(backendObjects)) + + // BaseObjects now includes services and gateway + result.BaseObjects = append(backendObjects, gatewayObjects...) + log.Printf("GenerateScaleHTTPRouteObjects: BaseObjects contains %d total objects", len(result.BaseObjects)) + + // Each iteration only creates one route for i := range numRoutes { r := route{ Name: fmt.Sprintf("route-%d", i), @@ -248,58 +304,59 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { BackendName: backendName, } - var listeners []listener - if i == 0 { - // only generate a Gateway on the first iteration - listeners = []listener{l} - } - - objects, err := generateManifests(listeners, []route{r}) + log.Printf("GenerateScaleHTTPRouteObjects: Generating route %d: %s with hostname %s", i, r.Name, r.HostnamePrefix) + // Generate only the route (no gateway, no listeners) + objects, err := generateManifests([]listener{}, []route{r}) if err != nil { return ScaleObjects{}, err } result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects) + log.Printf("GenerateScaleHTTPRouteObjects: Route %d generated %d objects", i, len(objects)) } - backendObjects, err := generateBackendAppObjects([]string{backendName}) - if err != nil { - return ScaleObjects{}, err - } - - result.BaseObjects = backendObjects - + log.Printf("GenerateScaleHTTPRouteObjects: Completed. BaseObjects: %d, ScaleIterationGroups: %d", len(result.BaseObjects), len(result.ScaleIterationGroups)) return result, nil } func generateManifests(listeners []listener, routes []route) ([]client.Object, error) { + log.Printf("generateManifests: Called with %d listeners, %d routes", len(listeners), len(routes)) var buf bytes.Buffer if len(listeners) > 0 { + log.Printf("generateManifests: Executing gateway template with %d listeners", len(listeners)) if err := gwTmpl.Execute(&buf, listeners); err != nil { return nil, err } } - for _, r := range routes { + for i, r := range routes { if buf.Len() > 0 { buf.WriteString("\n---\n") } + log.Printf("generateManifests: Executing route template for route %d: %s", i, r.Name) if err := hrTmpl.Execute(&buf, r); err != nil { return nil, err } } - return decodeObjects(&buf) + objects, err := decodeObjects(&buf) + if err != nil { + return nil, err + } + log.Printf("generateManifests: Decoded %d objects from templates", len(objects)) + return objects, nil } func generateBackendAppObjects(backends []string) ([]client.Object, error) { + log.Printf("generateBackendAppObjects: Generating objects for %d backends: %v", len(backends), backends) objects := make([]client.Object, 0, 2*len(backends)) - for _, backend := range backends { + for i, backend := range backends { var buf bytes.Buffer + log.Printf("generateBackendAppObjects: Executing app template for backend %d: %s", i, backend) if err := appTmpl.Execute(&buf, backend); err != nil { return nil, err } @@ -309,8 +366,10 @@ func generateBackendAppObjects(backends []string) ([]client.Object, error) { return nil, err } + log.Printf("generateBackendAppObjects: Backend %s generated %d objects", backend, len(objs)) objects = append(objects, objs...) } + log.Printf("generateBackendAppObjects: Total objects generated: %d", len(objects)) return objects, nil } diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index fba6430470..4e53588aa3 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -34,16 +34,16 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { // - Node: n2d-standard-16 (16 vCPU, 64GB memory) var ( - matchesManifests = []string{ - "scale/matches.yaml", - } - upstreamsManifests = []string{ - "scale/upstreams.yaml", - } + // matchesManifests = []string{ + // "scale/matches.yaml", + // } + // upstreamsManifests = []string{ + // "scale/upstreams.yaml", + // } - httpRouteManifests = []string{ - "scale/httproute.yaml", - } + // httpRouteManifests = []string{ + // "scale/httproute.yaml", + // } namespace = "scale" @@ -56,7 +56,7 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { promInstance framework.PrometheusInstance promPortForwardStopCh = make(chan struct{}) - upstreamServerCount int32 + // upstreamServerCount int32 ) const ( @@ -94,11 +94,11 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { Expect(promInstance.PortForward(k8sConfig, promPortForwardStopCh)).To(Succeed()) } - if *plusEnabled { - upstreamServerCount = plusUpstreamServerCount - } else { - upstreamServerCount = ossUpstreamServerCount - } + // if *plusEnabled { + // upstreamServerCount = plusUpstreamServerCount + // } else { + // upstreamServerCount = ossUpstreamServerCount + // } }) BeforeEach(func() { @@ -469,53 +469,53 @@ The logs are attached only if there are errors. Expect(os.Remove(ttrCsvFile.Name())).To(Succeed()) } - runScaleUpstreams := func() { - Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + // runScaleUpstreams := func() { + // Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) + // Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - // apply HTTPRoute after upstreams are ready - Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + // // apply HTTPRoute after upstreams are ready + // Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) + // Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - var nginxPodNames []string - var err error - Eventually( - func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) - return len(nginxPodNames) == 1 && err == nil - }). - WithTimeout(timeoutConfig.CreateTimeout). - Should(BeTrue()) + // var nginxPodNames []string + // var err error + // Eventually( + // func() bool { + // nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + // return len(nginxPodNames) == 1 && err == nil + // }). + // WithTimeout(timeoutConfig.CreateTimeout). + // Should(BeTrue()) - nginxPodName := nginxPodNames[0] - Expect(nginxPodName).ToNot(BeEmpty()) + // nginxPodName := nginxPodNames[0] + // Expect(nginxPodName).ToNot(BeEmpty()) - setUpPortForward(nginxPodName, namespace) + // setUpPortForward(nginxPodName, namespace) - var url string - if portFwdPort != 0 { - url = fmt.Sprintf("http://hello.example.com:%d", portFwdPort) - } else { - url = "http://hello.example.com" - } + // var url string + // if portFwdPort != 0 { + // url = fmt.Sprintf("http://hello.example.com:%d", portFwdPort) + // } else { + // url = "http://hello.example.com" + // } - Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), - ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) + // Eventually( + // framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + // ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) - Expect( - resourceManager.ScaleDeployment(namespace, "backend", upstreamServerCount), - ).To(Succeed()) + // Expect( + // resourceManager.ScaleDeployment(namespace, "backend", upstreamServerCount), + // ).To(Succeed()) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() + // ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + // defer cancel() - Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) + // Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) - Eventually( - framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), - ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) - } + // Eventually( + // framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + // ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) + // } setNamespace := func(objects framework.ScaleObjects) { for _, obj := range objects.BaseObjects { @@ -576,119 +576,119 @@ The logs are attached only if there are errors. ) }) - It(fmt.Sprintf("scales HTTP routes to %d", httpRouteCount), func() { - const testName = "TestScale_HTTPRoutes" - - testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) - - objects, err := framework.GenerateScaleHTTPRouteObjects(httpRouteCount) - Expect(err).ToNot(HaveOccurred()) - - setNamespace(objects) - - runTestWithMetricsAndLogs( - testName, - testResultsDir, - func() { - runScaleResources( - objects, - testResultsDir, - "http", - ) - }, - ) - }) - - It(fmt.Sprintf("scales upstream servers to %d for OSS and %d for Plus", - ossUpstreamServerCount, - plusUpstreamServerCount, - ), func() { - const testName = "TestScale_UpstreamServers" - - testResultsDir := filepath.Join(resultsDir, testName) - Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) - - runTestWithMetricsAndLogs( - testName, - testResultsDir, - func() { - runScaleUpstreams() - }, - ) - }) - - It("scales HTTP matches", func() { - const testName = "TestScale_HTTPMatches" - - Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) - Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - - var nginxPodNames []string - var err error - Eventually( - func() bool { - nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) - return len(nginxPodNames) == 1 && err == nil - }). - WithTimeout(timeoutConfig.CreateTimeout). - Should(BeTrue()) - - nginxPodName := nginxPodNames[0] - Expect(nginxPodName).ToNot(BeEmpty()) - - setUpPortForward(nginxPodName, namespace) - - var port int - if portFwdPort != 0 { - port = portFwdPort - } else { - port = 80 - } - - addr := fmt.Sprintf("%s:%d", address, port) - - baseURL := "http://cafe.example.com" - - text := fmt.Sprintf("\n## Test %s\n\n", testName) - - _, err = fmt.Fprint(outFile, text) - Expect(err).ToNot(HaveOccurred()) - - run := func(t framework.Target) { - cfg := framework.LoadTestConfig{ - Targets: []framework.Target{t}, - Rate: 1000, - Duration: 30 * time.Second, - Description: "First matches", - Proxy: addr, - ServerName: "cafe.example.com", - } - _, metrics := framework.RunLoadTest(cfg) - - _, err = fmt.Fprintln(outFile, "```text") - Expect(err).ToNot(HaveOccurred()) - Expect(framework.WriteMetricsResults(outFile, &metrics)).To(Succeed()) - _, err = fmt.Fprintln(outFile, "```") - Expect(err).ToNot(HaveOccurred()) - } - - run(framework.Target{ - Method: "GET", - URL: fmt.Sprintf("%s%s", baseURL, "/latte"), - Header: map[string][]string{ - "header-1": {"header-1-val"}, - }, - }) - - run(framework.Target{ - Method: "GET", - URL: fmt.Sprintf("%s%s", baseURL, "/latte"), - Header: map[string][]string{ - "header-50": {"header-50-val"}, - }, - }) - }) + // It(fmt.Sprintf("scales HTTP routes to %d", httpRouteCount), func() { + // const testName = "TestScale_HTTPRoutes" + + // testResultsDir := filepath.Join(resultsDir, testName) + // Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + + // objects, err := framework.GenerateScaleHTTPRouteObjects(httpRouteCount) + // Expect(err).ToNot(HaveOccurred()) + + // setNamespace(objects) + + // runTestWithMetricsAndLogs( + // testName, + // testResultsDir, + // func() { + // runScaleResources( + // objects, + // testResultsDir, + // "http", + // ) + // }, + // ) + // }) + + // It(fmt.Sprintf("scales upstream servers to %d for OSS and %d for Plus", + // ossUpstreamServerCount, + // plusUpstreamServerCount, + // ), func() { + // const testName = "TestScale_UpstreamServers" + + // testResultsDir := filepath.Join(resultsDir, testName) + // Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + + // runTestWithMetricsAndLogs( + // testName, + // testResultsDir, + // func() { + // runScaleUpstreams() + // }, + // ) + // }) + + // It("scales HTTP matches", func() { + // const testName = "TestScale_HTTPMatches" + + // Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) + // Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + + // var nginxPodNames []string + // var err error + // Eventually( + // func() bool { + // nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + // return len(nginxPodNames) == 1 && err == nil + // }). + // WithTimeout(timeoutConfig.CreateTimeout). + // Should(BeTrue()) + + // nginxPodName := nginxPodNames[0] + // Expect(nginxPodName).ToNot(BeEmpty()) + + // setUpPortForward(nginxPodName, namespace) + + // var port int + // if portFwdPort != 0 { + // port = portFwdPort + // } else { + // port = 80 + // } + + // addr := fmt.Sprintf("%s:%d", address, port) + + // baseURL := "http://cafe.example.com" + + // text := fmt.Sprintf("\n## Test %s\n\n", testName) + + // _, err = fmt.Fprint(outFile, text) + // Expect(err).ToNot(HaveOccurred()) + + // run := func(t framework.Target) { + // cfg := framework.LoadTestConfig{ + // Targets: []framework.Target{t}, + // Rate: 1000, + // Duration: 30 * time.Second, + // Description: "First matches", + // Proxy: addr, + // ServerName: "cafe.example.com", + // } + // _, metrics := framework.RunLoadTest(cfg) + + // _, err = fmt.Fprintln(outFile, "```text") + // Expect(err).ToNot(HaveOccurred()) + // Expect(framework.WriteMetricsResults(outFile, &metrics)).To(Succeed()) + // _, err = fmt.Fprintln(outFile, "```") + // Expect(err).ToNot(HaveOccurred()) + // } + + // run(framework.Target{ + // Method: "GET", + // URL: fmt.Sprintf("%s%s", baseURL, "/latte"), + // Header: map[string][]string{ + // "header-1": {"header-1-val"}, + // }, + // }) + + // run(framework.Target{ + // Method: "GET", + // URL: fmt.Sprintf("%s%s", baseURL, "/latte"), + // Header: map[string][]string{ + // "header-50": {"header-50-val"}, + // }, + // }) + // }) AfterEach(func() { framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) From 92f091841994fe4ca9a0c3dba34d5e026e2864ce Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Mon, 8 Sep 2025 16:15:25 -0600 Subject: [PATCH 7/8] generate manifests fix --- tests/framework/generate_manifests.go | 112 +++------ tests/framework/results.go | 6 + tests/suite/scale_test.go | 337 +++++++++++++------------- 3 files changed, 212 insertions(+), 243 deletions(-) diff --git a/tests/framework/generate_manifests.go b/tests/framework/generate_manifests.go index e4a2d05a9a..8e5d436fd6 100644 --- a/tests/framework/generate_manifests.go +++ b/tests/framework/generate_manifests.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "log" "text/template" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -121,9 +120,14 @@ type route struct { // ScaleObjects contains objects for scale testing. type ScaleObjects struct { - // BaseObjects contains objects that are common to all scale iterations. + // BaseObjects contains objects that should be created first: + // secrets and other foundational resources. BaseObjects []client.Object - // ScaleIterationGroups contains objects for each scale iteration. + // GatewayAndServiceObjects contains backend services, deployments, and Gateway objects. + // These are created after BaseObjects to ensure endpoints are ready before traffic. + GatewayAndServiceObjects []client.Object + // ScaleIterationGroups contains HTTPRoute objects for each scale iteration. + // These are applied after GatewayAndServiceObjects to start traffic flow incrementally. ScaleIterationGroups [][]client.Object } @@ -151,9 +155,9 @@ func decodeObjects(reader io.Reader) ([]client.Object, error) { } // GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test. -// Modified to create all services and gateways first, then scale routes. +// Secrets are created first in BaseObjects, then backend services/deployments and Gateway in GatewayAndServiceObjects, +// and finally HTTPRoutes in ScaleIterationGroups. func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) { - log.Printf("GenerateScaleListenerObjects: Starting generation for %d listeners, TLS=%v", numListeners, tls) var result ScaleObjects listeners := make([]listener, 0) @@ -161,7 +165,6 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err secrets := make([]string, 0) routes := make([]route, 0) - // Generate all listeners, backends, and routes for i := range numListeners { listenerName := fmt.Sprintf("listener-%d", i) hostnamePrefix := fmt.Sprintf("%d", i) @@ -185,66 +188,44 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err HostnamePrefix: hostnamePrefix, BackendName: backendName, } - routes = append(routes, r) + backends = append(backends, backendName) } - log.Printf("GenerateScaleListenerObjects: Generated %d listeners, %d backends, %d secrets", len(listeners), len(backends), len(secrets)) - - // Generate gateway with all listeners - log.Printf("GenerateScaleListenerObjects: Generating gateway with %d listeners", len(listeners)) - gatewayObjects, err := generateManifests(listeners, []route{}) + secretObjects, err := generateSecrets(secrets) if err != nil { return ScaleObjects{}, err } - log.Printf("GenerateScaleListenerObjects: Generated %d gateway objects", len(gatewayObjects)) + result.BaseObjects = append(result.BaseObjects, secretObjects...) - // Generate secrets if TLS is enabled - secretObjects, err := generateSecrets(secrets) + backendObjects, err := generateBackendAppObjects(backends) if err != nil { return ScaleObjects{}, err } - log.Printf("GenerateScaleListenerObjects: Generated %d secret objects", len(secretObjects)) + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, backendObjects...) - // Generate backend app objects (services, deployments) - log.Printf("GenerateScaleListenerObjects: Generating backend objects for %d backends", len(backends)) - backendObjects, err := generateBackendAppObjects(backends) + gatewayObjects, err := generateManifests(listeners, nil) if err != nil { return ScaleObjects{}, err } - log.Printf("GenerateScaleListenerObjects: Generated %d backend objects", len(backendObjects)) - - // BaseObjects includes services, secrets, and gateway with all listeners - result.BaseObjects = append(result.BaseObjects, secretObjects...) - result.BaseObjects = append(result.BaseObjects, backendObjects...) - result.BaseObjects = append(result.BaseObjects, gatewayObjects...) - log.Printf("GenerateScaleListenerObjects: BaseObjects contains %d total objects", len(result.BaseObjects)) + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, gatewayObjects...) - // Each iteration creates one route - for i := range numListeners { - log.Printf("GenerateScaleListenerObjects: Generating route %d: %s", i, routes[i].Name) - objects, err := generateManifests([]listener{}, []route{routes[i]}) - if err != nil { - return ScaleObjects{}, err - } - - result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects) - log.Printf("GenerateScaleListenerObjects: Route %d generated %d objects", i, len(objects)) + routeObjects, err := generateManifests(nil, routes) + if err != nil { + return ScaleObjects{}, err } + result.ScaleIterationGroups = append(result.ScaleIterationGroups, routeObjects) - log.Printf("GenerateScaleListenerObjects: Completed. BaseObjects: %d, ScaleIterationGroups: %d", len(result.BaseObjects), len(result.ScaleIterationGroups)) return result, nil } func generateSecrets(secrets []string) ([]client.Object, error) { - log.Printf("generateSecrets: Generating %d secrets: %v", len(secrets), secrets) objects := make([]client.Object, 0, len(secrets)) - for i, secret := range secrets { + for _, secret := range secrets { var buf bytes.Buffer - log.Printf("generateSecrets: Executing secret template for secret %d: %s", i, secret) if err := secretTmpl.Execute(&buf, secret); err != nil { return nil, err } @@ -254,18 +235,14 @@ func generateSecrets(secrets []string) ([]client.Object, error) { return nil, err } - log.Printf("generateSecrets: Secret %s generated %d objects", secret, len(objs)) objects = append(objects, objs...) } - log.Printf("generateSecrets: Total secret objects generated: %d", len(objects)) return objects, nil } // GenerateScaleHTTPRouteObjects generates objects for a given number of routes for the scale test. -// Modified to create all services and gateways first, then scale routes. func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { - log.Printf("GenerateScaleHTTPRouteObjects: Starting generation for %d routes", numRoutes) var result ScaleObjects l := listener{ @@ -275,27 +252,21 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { backendName := "backend" - // Generate the Gateway once and add it to BaseObjects - log.Printf("GenerateScaleHTTPRouteObjects: Generating gateway with single listener") - gatewayObjects, err := generateManifests([]listener{l}, []route{}) + // Generate backend objects and add to GatewayAndServiceObjects + backendObjects, err := generateBackendAppObjects([]string{backendName}) if err != nil { return ScaleObjects{}, err } - log.Printf("GenerateScaleHTTPRouteObjects: Generated %d gateway objects", len(gatewayObjects)) + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, backendObjects...) - // Generate backend app objects (services, deployments) - log.Printf("GenerateScaleHTTPRouteObjects: Generating backend objects for backend: %s", backendName) - backendObjects, err := generateBackendAppObjects([]string{backendName}) + // Generate Gateway object and add to GatewayAndServiceObjects + gatewayObjects, err := generateManifests([]listener{l}, nil) if err != nil { return ScaleObjects{}, err } - log.Printf("GenerateScaleHTTPRouteObjects: Generated %d backend objects", len(backendObjects)) - - // BaseObjects now includes services and gateway - result.BaseObjects = append(backendObjects, gatewayObjects...) - log.Printf("GenerateScaleHTTPRouteObjects: BaseObjects contains %d total objects", len(result.BaseObjects)) + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, gatewayObjects...) - // Each iteration only creates one route + // Generate HTTPRoute objects for each iteration for i := range numRoutes { r := route{ Name: fmt.Sprintf("route-%d", i), @@ -304,59 +275,46 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { BackendName: backendName, } - log.Printf("GenerateScaleHTTPRouteObjects: Generating route %d: %s with hostname %s", i, r.Name, r.HostnamePrefix) - // Generate only the route (no gateway, no listeners) - objects, err := generateManifests([]listener{}, []route{r}) + // Generate only the HTTPRoute (no listeners/gateway) + routeObjects, err := generateManifests(nil, []route{r}) if err != nil { return ScaleObjects{}, err } - result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects) - log.Printf("GenerateScaleHTTPRouteObjects: Route %d generated %d objects", i, len(objects)) + result.ScaleIterationGroups = append(result.ScaleIterationGroups, routeObjects) } - log.Printf("GenerateScaleHTTPRouteObjects: Completed. BaseObjects: %d, ScaleIterationGroups: %d", len(result.BaseObjects), len(result.ScaleIterationGroups)) return result, nil } func generateManifests(listeners []listener, routes []route) ([]client.Object, error) { - log.Printf("generateManifests: Called with %d listeners, %d routes", len(listeners), len(routes)) var buf bytes.Buffer if len(listeners) > 0 { - log.Printf("generateManifests: Executing gateway template with %d listeners", len(listeners)) if err := gwTmpl.Execute(&buf, listeners); err != nil { return nil, err } } - for i, r := range routes { + for _, r := range routes { if buf.Len() > 0 { buf.WriteString("\n---\n") } - log.Printf("generateManifests: Executing route template for route %d: %s", i, r.Name) if err := hrTmpl.Execute(&buf, r); err != nil { return nil, err } } - objects, err := decodeObjects(&buf) - if err != nil { - return nil, err - } - log.Printf("generateManifests: Decoded %d objects from templates", len(objects)) - return objects, nil + return decodeObjects(&buf) } func generateBackendAppObjects(backends []string) ([]client.Object, error) { - log.Printf("generateBackendAppObjects: Generating objects for %d backends: %v", len(backends), backends) objects := make([]client.Object, 0, 2*len(backends)) - for i, backend := range backends { + for _, backend := range backends { var buf bytes.Buffer - log.Printf("generateBackendAppObjects: Executing app template for backend %d: %s", i, backend) if err := appTmpl.Execute(&buf, backend); err != nil { return nil, err } @@ -366,10 +324,8 @@ func generateBackendAppObjects(backends []string) ([]client.Object, error) { return nil, err } - log.Printf("generateBackendAppObjects: Backend %s generated %d objects", backend, len(objs)) objects = append(objects, objs...) } - log.Printf("generateBackendAppObjects: Total objects generated: %d", len(objects)) return objects, nil } diff --git a/tests/framework/results.go b/tests/framework/results.go index 97a4c5d57a..120dbcc3a5 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -21,6 +21,12 @@ func CreateResultsDir(testName, version string) (string, error) { dirName := filepath.Join(filepath.Dir(pwd), "results", testName, version) + if _, err := os.Stat(dirName); err == nil { + if err := os.RemoveAll(dirName); err != nil { + return "", fmt.Errorf("failed to remove existing directory %s: %w", dirName, err) + } + } + return dirName, os.MkdirAll(dirName, 0o777) } diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 4e53588aa3..a15b5fa443 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -34,16 +34,16 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { // - Node: n2d-standard-16 (16 vCPU, 64GB memory) var ( - // matchesManifests = []string{ - // "scale/matches.yaml", - // } - // upstreamsManifests = []string{ - // "scale/upstreams.yaml", - // } + matchesManifests = []string{ + "scale/matches.yaml", + } + upstreamsManifests = []string{ + "scale/upstreams.yaml", + } - // httpRouteManifests = []string{ - // "scale/httproute.yaml", - // } + httpRouteManifests = []string{ + "scale/httproute.yaml", + } namespace = "scale" @@ -56,7 +56,7 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { promInstance framework.PrometheusInstance promPortForwardStopCh = make(chan struct{}) - // upstreamServerCount int32 + upstreamServerCount int32 ) const ( @@ -94,11 +94,11 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { Expect(promInstance.PortForward(k8sConfig, promPortForwardStopCh)).To(Succeed()) } - // if *plusEnabled { - // upstreamServerCount = plusUpstreamServerCount - // } else { - // upstreamServerCount = ossUpstreamServerCount - // } + if *plusEnabled { + upstreamServerCount = plusUpstreamServerCount + } else { + upstreamServerCount = ossUpstreamServerCount + } }) BeforeEach(func() { @@ -409,8 +409,12 @@ The logs are attached only if there are errors. Expect(err).ToNot(HaveOccurred()) defer ttrCsvFile.Close() + // Apply BaseObjects first (secrets and other foundational resources) Expect(resourceManager.Apply(objects.BaseObjects)).To(Succeed()) + // Apply GatewayAndServiceObjects next (backend services, deployments, and Gateway) + Expect(resourceManager.Apply(objects.GatewayAndServiceObjects)).To(Succeed()) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() @@ -469,58 +473,61 @@ The logs are attached only if there are errors. Expect(os.Remove(ttrCsvFile.Name())).To(Succeed()) } - // runScaleUpstreams := func() { - // Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) - // Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + runScaleUpstreams := func() { + Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - // // apply HTTPRoute after upstreams are ready - // Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) - // Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + // apply HTTPRoute after upstreams are ready + Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - // var nginxPodNames []string - // var err error - // Eventually( - // func() bool { - // nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) - // return len(nginxPodNames) == 1 && err == nil - // }). - // WithTimeout(timeoutConfig.CreateTimeout). - // Should(BeTrue()) + var nginxPodNames []string + var err error + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) - // nginxPodName := nginxPodNames[0] - // Expect(nginxPodName).ToNot(BeEmpty()) + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) - // setUpPortForward(nginxPodName, namespace) + setUpPortForward(nginxPodName, namespace) - // var url string - // if portFwdPort != 0 { - // url = fmt.Sprintf("http://hello.example.com:%d", portFwdPort) - // } else { - // url = "http://hello.example.com" - // } + var url string + if portFwdPort != 0 { + url = fmt.Sprintf("http://hello.example.com:%d", portFwdPort) + } else { + url = "http://hello.example.com" + } - // Eventually( - // framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), - // ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) + Eventually( + framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) - // Expect( - // resourceManager.ScaleDeployment(namespace, "backend", upstreamServerCount), - // ).To(Succeed()) + Expect( + resourceManager.ScaleDeployment(namespace, "backend", upstreamServerCount), + ).To(Succeed()) - // ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - // defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() - // Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) + Expect(resourceManager.WaitForPodsToBeReady(ctx, namespace)).To(Succeed()) - // Eventually( - // framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), - // ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) - // } + Eventually( + framework.CreateResponseChecker(url, address, timeoutConfig.RequestTimeout), + ).WithTimeout(5 * timeoutConfig.RequestTimeout).WithPolling(100 * time.Millisecond).Should(Succeed()) + } setNamespace := func(objects framework.ScaleObjects) { for _, obj := range objects.BaseObjects { obj.SetNamespace(namespace) } + for _, obj := range objects.GatewayAndServiceObjects { + obj.SetNamespace(namespace) + } for _, objs := range objects.ScaleIterationGroups { for _, obj := range objs { obj.SetNamespace(namespace) @@ -576,119 +583,119 @@ The logs are attached only if there are errors. ) }) - // It(fmt.Sprintf("scales HTTP routes to %d", httpRouteCount), func() { - // const testName = "TestScale_HTTPRoutes" - - // testResultsDir := filepath.Join(resultsDir, testName) - // Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) - - // objects, err := framework.GenerateScaleHTTPRouteObjects(httpRouteCount) - // Expect(err).ToNot(HaveOccurred()) - - // setNamespace(objects) - - // runTestWithMetricsAndLogs( - // testName, - // testResultsDir, - // func() { - // runScaleResources( - // objects, - // testResultsDir, - // "http", - // ) - // }, - // ) - // }) - - // It(fmt.Sprintf("scales upstream servers to %d for OSS and %d for Plus", - // ossUpstreamServerCount, - // plusUpstreamServerCount, - // ), func() { - // const testName = "TestScale_UpstreamServers" - - // testResultsDir := filepath.Join(resultsDir, testName) - // Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) - - // runTestWithMetricsAndLogs( - // testName, - // testResultsDir, - // func() { - // runScaleUpstreams() - // }, - // ) - // }) - - // It("scales HTTP matches", func() { - // const testName = "TestScale_HTTPMatches" - - // Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) - // Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) - - // var nginxPodNames []string - // var err error - // Eventually( - // func() bool { - // nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) - // return len(nginxPodNames) == 1 && err == nil - // }). - // WithTimeout(timeoutConfig.CreateTimeout). - // Should(BeTrue()) - - // nginxPodName := nginxPodNames[0] - // Expect(nginxPodName).ToNot(BeEmpty()) - - // setUpPortForward(nginxPodName, namespace) - - // var port int - // if portFwdPort != 0 { - // port = portFwdPort - // } else { - // port = 80 - // } - - // addr := fmt.Sprintf("%s:%d", address, port) - - // baseURL := "http://cafe.example.com" - - // text := fmt.Sprintf("\n## Test %s\n\n", testName) - - // _, err = fmt.Fprint(outFile, text) - // Expect(err).ToNot(HaveOccurred()) - - // run := func(t framework.Target) { - // cfg := framework.LoadTestConfig{ - // Targets: []framework.Target{t}, - // Rate: 1000, - // Duration: 30 * time.Second, - // Description: "First matches", - // Proxy: addr, - // ServerName: "cafe.example.com", - // } - // _, metrics := framework.RunLoadTest(cfg) - - // _, err = fmt.Fprintln(outFile, "```text") - // Expect(err).ToNot(HaveOccurred()) - // Expect(framework.WriteMetricsResults(outFile, &metrics)).To(Succeed()) - // _, err = fmt.Fprintln(outFile, "```") - // Expect(err).ToNot(HaveOccurred()) - // } - - // run(framework.Target{ - // Method: "GET", - // URL: fmt.Sprintf("%s%s", baseURL, "/latte"), - // Header: map[string][]string{ - // "header-1": {"header-1-val"}, - // }, - // }) - - // run(framework.Target{ - // Method: "GET", - // URL: fmt.Sprintf("%s%s", baseURL, "/latte"), - // Header: map[string][]string{ - // "header-50": {"header-50-val"}, - // }, - // }) - // }) + It(fmt.Sprintf("scales HTTP routes to %d", httpRouteCount), func() { + const testName = "TestScale_HTTPRoutes" + + testResultsDir := filepath.Join(resultsDir, testName) + Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + + objects, err := framework.GenerateScaleHTTPRouteObjects(httpRouteCount) + Expect(err).ToNot(HaveOccurred()) + + setNamespace(objects) + + runTestWithMetricsAndLogs( + testName, + testResultsDir, + func() { + runScaleResources( + objects, + testResultsDir, + "http", + ) + }, + ) + }) + + It(fmt.Sprintf("scales upstream servers to %d for OSS and %d for Plus", + ossUpstreamServerCount, + plusUpstreamServerCount, + ), func() { + const testName = "TestScale_UpstreamServers" + + testResultsDir := filepath.Join(resultsDir, testName) + Expect(os.MkdirAll(testResultsDir, 0o755)).To(Succeed()) + + runTestWithMetricsAndLogs( + testName, + testResultsDir, + func() { + runScaleUpstreams() + }, + ) + }) + + It("scales HTTP matches", func() { + const testName = "TestScale_HTTPMatches" + + Expect(resourceManager.ApplyFromFiles(matchesManifests, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + + var nginxPodNames []string + var err error + Eventually( + func() bool { + nginxPodNames, err = framework.GetReadyNginxPodNames(k8sClient, namespace, timeoutConfig.GetStatusTimeout) + return len(nginxPodNames) == 1 && err == nil + }). + WithTimeout(timeoutConfig.CreateTimeout). + Should(BeTrue()) + + nginxPodName := nginxPodNames[0] + Expect(nginxPodName).ToNot(BeEmpty()) + + setUpPortForward(nginxPodName, namespace) + + var port int + if portFwdPort != 0 { + port = portFwdPort + } else { + port = 80 + } + + addr := fmt.Sprintf("%s:%d", address, port) + + baseURL := "http://cafe.example.com" + + text := fmt.Sprintf("\n## Test %s\n\n", testName) + + _, err = fmt.Fprint(outFile, text) + Expect(err).ToNot(HaveOccurred()) + + run := func(t framework.Target) { + cfg := framework.LoadTestConfig{ + Targets: []framework.Target{t}, + Rate: 1000, + Duration: 30 * time.Second, + Description: "First matches", + Proxy: addr, + ServerName: "cafe.example.com", + } + _, metrics := framework.RunLoadTest(cfg) + + _, err = fmt.Fprintln(outFile, "```text") + Expect(err).ToNot(HaveOccurred()) + Expect(framework.WriteMetricsResults(outFile, &metrics)).To(Succeed()) + _, err = fmt.Fprintln(outFile, "```") + Expect(err).ToNot(HaveOccurred()) + } + + run(framework.Target{ + Method: "GET", + URL: fmt.Sprintf("%s%s", baseURL, "/latte"), + Header: map[string][]string{ + "header-1": {"header-1-val"}, + }, + }) + + run(framework.Target{ + Method: "GET", + URL: fmt.Sprintf("%s%s", baseURL, "/latte"), + Header: map[string][]string{ + "header-50": {"header-50-val"}, + }, + }) + }) AfterEach(func() { framework.AddNginxLogsAndEventsToReport(resourceManager, namespace) From 9647d52e75eb3fac1dff1e85fc2986f7b7f9485e Mon Sep 17 00:00:00 2001 From: salonichf5 <146118978+salonichf5@users.noreply.github.com> Date: Tue, 9 Sep 2025 09:08:16 -0600 Subject: [PATCH 8/8] revert file close check --- tests/suite/scale_test.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index a15b5fa443..dfe9a24978 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "errors" "fmt" "io" "os" @@ -851,16 +850,7 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim AfterAll(func() { _, err := fmt.Fprint(outFile) Expect(err).ToNot(HaveOccurred()) - - if outFile != nil { - if err := outFile.Close(); err != nil { - if errors.Is(err, os.ErrClosed) || strings.Contains(err.Error(), "file already closed") { - GinkgoWriter.Printf("Warning: attempted to close already closed file: %v\n", err) - } else { - Expect(err).ToNot(HaveOccurred()) - } - } - } + Expect(outFile.Close()).To(Succeed()) // restoring NGF shared among tests in the suite cfg := getDefaultSetupCfg()