diff --git a/Makefile b/Makefile index cc408afc54..c794b3a9e2 100644 --- a/Makefile +++ b/Makefile @@ -130,7 +130,7 @@ install-crds: ## Install CRDs .PHONY: install-gateway-crds install-gateway-crds: ## Install Gateway API CRDs - kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply -f - + kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply --server-side -f - .PHONY: uninstall-gateway-crds uninstall-gateway-crds: ## Uninstall Gateway API CRDs diff --git a/tests/Makefile b/tests/Makefile index f1ccfdd3c5..668fd879e3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -122,7 +122,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) CGO_ENABLED=1 go run github.com/onsi/ginkgo/v2/ginkgo --race --randomize-all --randomize-suites --keep-going --fail-on-pending \ - --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) --flake-attempts=2 \ + --trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) --flake-attempts=1 --show-node-events --output-interceptor-mode=none \ --label-filter "nfr" $(GINKGO_FLAGS) --timeout 5h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ --gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ @@ -176,7 +176,7 @@ HELM_PARAMETERS += --set nginxGateway.name=nginx-gateway --set nginx.service.typ # it overrides the target in the main Makefile when the GW_API_VERSION is set to main ifeq ($(GW_API_VERSION),main) install-gateway-crds: - kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply -f - + kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply --server-side -f - endif .PHONY: install-ngf-local-no-build diff --git a/tests/framework/generate_manifests.go b/tests/framework/generate_manifests.go index e0ad091585..8e5d436fd6 100644 --- a/tests/framework/generate_manifests.go +++ b/tests/framework/generate_manifests.go @@ -120,9 +120,14 @@ type route struct { // ScaleObjects contains objects for scale testing. type ScaleObjects struct { - // BaseObjects contains objects that are common to all scale iterations. + // BaseObjects contains objects that should be created first: + // secrets and other foundational resources. BaseObjects []client.Object - // ScaleIterationGroups contains objects for each scale iteration. + // GatewayAndServiceObjects contains backend services, deployments, and Gateway objects. + // These are created after BaseObjects to ensure endpoints are ready before traffic. + GatewayAndServiceObjects []client.Object + // ScaleIterationGroups contains HTTPRoute objects for each scale iteration. + // These are applied after GatewayAndServiceObjects to start traffic flow incrementally. ScaleIterationGroups [][]client.Object } @@ -150,12 +155,15 @@ func decodeObjects(reader io.Reader) ([]client.Object, error) { } // GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test. +// Secrets are created first in BaseObjects, then backend services/deployments and Gateway in GatewayAndServiceObjects, +// and finally HTTPRoutes in ScaleIterationGroups. func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) { var result ScaleObjects listeners := make([]listener, 0) backends := make([]string, 0) secrets := make([]string, 0) + routes := make([]route, 0) for i := range numListeners { listenerName := fmt.Sprintf("listener-%d", i) @@ -180,30 +188,34 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err HostnamePrefix: hostnamePrefix, BackendName: backendName, } + routes = append(routes, r) backends = append(backends, backendName) - - objects, err := generateManifests(listeners, []route{r}) - if err != nil { - return ScaleObjects{}, err - } - - result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects) } secretObjects, err := generateSecrets(secrets) if err != nil { return ScaleObjects{}, err } - result.BaseObjects = append(result.BaseObjects, secretObjects...) backendObjects, err := generateBackendAppObjects(backends) if err != nil { return ScaleObjects{}, err } + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, backendObjects...) - result.BaseObjects = append(result.BaseObjects, backendObjects...) + gatewayObjects, err := generateManifests(listeners, nil) + if err != nil { + return ScaleObjects{}, err + } + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, gatewayObjects...) + + routeObjects, err := generateManifests(nil, routes) + if err != nil { + return ScaleObjects{}, err + } + result.ScaleIterationGroups = append(result.ScaleIterationGroups, routeObjects) return result, nil } @@ -240,6 +252,21 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { backendName := "backend" + // Generate backend objects and add to GatewayAndServiceObjects + backendObjects, err := generateBackendAppObjects([]string{backendName}) + if err != nil { + return ScaleObjects{}, err + } + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, backendObjects...) + + // Generate Gateway object and add to GatewayAndServiceObjects + gatewayObjects, err := generateManifests([]listener{l}, nil) + if err != nil { + return ScaleObjects{}, err + } + result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, gatewayObjects...) + + // Generate HTTPRoute objects for each iteration for i := range numRoutes { r := route{ Name: fmt.Sprintf("route-%d", i), @@ -248,27 +275,15 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) { BackendName: backendName, } - var listeners []listener - if i == 0 { - // only generate a Gateway on the first iteration - listeners = []listener{l} - } - - objects, err := generateManifests(listeners, []route{r}) + // Generate only the HTTPRoute (no listeners/gateway) + routeObjects, err := generateManifests(nil, []route{r}) if err != nil { return ScaleObjects{}, err } - result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects) + result.ScaleIterationGroups = append(result.ScaleIterationGroups, routeObjects) } - backendObjects, err := generateBackendAppObjects([]string{backendName}) - if err != nil { - return ScaleObjects{}, err - } - - result.BaseObjects = backendObjects - return result, nil } diff --git a/tests/framework/results.go b/tests/framework/results.go index 97a4c5d57a..120dbcc3a5 100644 --- a/tests/framework/results.go +++ b/tests/framework/results.go @@ -21,6 +21,12 @@ func CreateResultsDir(testName, version string) (string, error) { dirName := filepath.Join(filepath.Dir(pwd), "results", testName, version) + if _, err := os.Stat(dirName); err == nil { + if err := os.RemoveAll(dirName); err != nil { + return "", fmt.Errorf("failed to remove existing directory %s: %w", dirName, err) + } + } + return dirName, os.MkdirAll(dirName, 0o777) } diff --git a/tests/framework/timeout.go b/tests/framework/timeout.go index 394f9e351d..7c47b191bd 100644 --- a/tests/framework/timeout.go +++ b/tests/framework/timeout.go @@ -38,22 +38,26 @@ type TimeoutConfig struct { // KubernetesClientTimeout represents the maximum time for Kubernetes client operations. KubernetesClientTimeout time.Duration + + // GatewayListenerUpdateTimeout represents the maximum time for Gateway Listener count to be updated. + GatewayListenerUpdateTimeout time.Duration } // DefaultTimeoutConfig populates a TimeoutConfig with the default values. func DefaultTimeoutConfig() TimeoutConfig { return TimeoutConfig{ - CreateTimeout: 60 * time.Second, - UpdateTimeout: 60 * time.Second, - DeleteTimeout: 10 * time.Second, - DeleteNamespaceTimeout: 150 * time.Second, - GetTimeout: 10 * time.Second, - ManifestFetchTimeout: 10 * time.Second, - RequestTimeout: 10 * time.Second, - ContainerRestartTimeout: 10 * time.Second, - GetLeaderLeaseTimeout: 60 * time.Second, - GetStatusTimeout: 60 * time.Second, - TestForTrafficTimeout: 60 * time.Second, - KubernetesClientTimeout: 10 * time.Second, + CreateTimeout: 60 * time.Second, + UpdateTimeout: 60 * time.Second, + DeleteTimeout: 10 * time.Second, + DeleteNamespaceTimeout: 150 * time.Second, + GetTimeout: 10 * time.Second, + ManifestFetchTimeout: 10 * time.Second, + RequestTimeout: 30 * time.Second, + ContainerRestartTimeout: 10 * time.Second, + GetLeaderLeaseTimeout: 60 * time.Second, + GetStatusTimeout: 60 * time.Second, + TestForTrafficTimeout: 60 * time.Second, + KubernetesClientTimeout: 10 * time.Second, + GatewayListenerUpdateTimeout: 60 * time.Second, } } diff --git a/tests/suite/manifests/scale/httproute.yaml b/tests/suite/manifests/scale/httproute.yaml new file mode 100644 index 0000000000..51ecb72cb6 --- /dev/null +++ b/tests/suite/manifests/scale/httproute.yaml @@ -0,0 +1,18 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: route +spec: + parentRefs: + - name: gateway + sectionName: listener + hostnames: + - "*.example.com" + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: backend + port: 80 diff --git a/tests/suite/manifests/scale/upstreams.yaml b/tests/suite/manifests/scale/upstreams.yaml index 1c9587f31c..0626024f93 100644 --- a/tests/suite/manifests/scale/upstreams.yaml +++ b/tests/suite/manifests/scale/upstreams.yaml @@ -13,25 +13,6 @@ spec: port: 80 protocol: HTTP --- -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: route -spec: - parentRefs: - - name: gateway - sectionName: listener - hostnames: - - "*.example.com" - rules: - - matches: - - path: - type: PathPrefix - value: / - backendRefs: - - name: backend - port: 80 ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/tests/suite/scale_test.go b/tests/suite/scale_test.go index 482740b726..dfe9a24978 100644 --- a/tests/suite/scale_test.go +++ b/tests/suite/scale_test.go @@ -40,6 +40,10 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() { "scale/upstreams.yaml", } + httpRouteManifests = []string{ + "scale/httproute.yaml", + } + namespace = "scale" scrapeInterval = 15 * time.Second @@ -404,8 +408,12 @@ The logs are attached only if there are errors. Expect(err).ToNot(HaveOccurred()) defer ttrCsvFile.Close() + // Apply BaseObjects first (secrets and other foundational resources) Expect(resourceManager.Apply(objects.BaseObjects)).To(Succeed()) + // Apply GatewayAndServiceObjects next (backend services, deployments, and Gateway) + Expect(resourceManager.Apply(objects.GatewayAndServiceObjects)).To(Succeed()) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() @@ -468,6 +476,10 @@ The logs are attached only if there are errors. Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed()) Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + // apply HTTPRoute after upstreams are ready + Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed()) + Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed()) + var nginxPodNames []string var err error Eventually( @@ -512,6 +524,9 @@ The logs are attached only if there are errors. for _, obj := range objects.BaseObjects { obj.SetNamespace(namespace) } + for _, obj := range objects.GatewayAndServiceObjects { + obj.SetNamespace(namespace) + } for _, objs := range objects.ScaleIterationGroups { for _, obj := range objs { obj.SetNamespace(namespace) @@ -1012,13 +1027,13 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim checkGatewayListeners := func(num int) { Eventually( func() error { - ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout) + ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout*2) defer cancel() var gw v1.Gateway key := types.NamespacedName{Namespace: ns.Name, Name: "gateway"} if err := resourceManager.K8sClient.Get(ctx, key, &gw); err != nil { - return err + return fmt.Errorf("failed to get gateway: %w", err) } if len(gw.Status.Listeners) != num { @@ -1028,8 +1043,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim return nil }, ). - WithTimeout(5 * time.Second). - WithPolling(100 * time.Millisecond). + WithTimeout(timeoutConfig.GatewayListenerUpdateTimeout). + WithPolling(1 * time.Second). Should(Succeed()) }