Fix some of the NFR tests to reduce error logs and pipeline failures (#3827)

Reduce NFR tests error logs and pipeline failures

Author: salonichf5

Makefile

@@ -130,7 +130,7 @@ install-crds: ## Install CRDs
 
 .PHONY: install-gateway-crds
 install-gateway-crds: ## Install Gateway API CRDs
-	kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply -f -
+	kubectl kustomize $(SELF_DIR)config/crd/gateway-api/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,standard) | kubectl apply --server-side -f -
 
 .PHONY: uninstall-gateway-crds
 uninstall-gateway-crds: ## Uninstall Gateway API CRDs

tests/Makefile

@@ -122,7 +122,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 .PHONY: .vm-nfr-test
 .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
 	CGO_ENABLED=1 go run github.com/onsi/ginkgo/v2/ginkgo --race --randomize-all --randomize-suites --keep-going --fail-on-pending \
-		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) --flake-attempts=2 \
+		--trace -r -v --buildvcs --force-newlines $(GITHUB_OUTPUT) --flake-attempts=1 --show-node-events --output-interceptor-mode=none \
 		--label-filter "nfr" $(GINKGO_FLAGS) --timeout 5h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
@@ -176,7 +176,7 @@ HELM_PARAMETERS += --set nginxGateway.name=nginx-gateway --set nginx.service.typ
 # it overrides the target in the main Makefile when the GW_API_VERSION is set to main
 ifeq ($(GW_API_VERSION),main)
 install-gateway-crds:
-	kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply -f -
+	kubectl kustomize "https://github.com/kubernetes-sigs/gateway-api/config/crd/$(if $(filter true,$(ENABLE_EXPERIMENTAL)),experimental,)?timeout=120&ref=main" | kubectl apply --server-side -f -
 endif
 
 .PHONY: install-ngf-local-no-build

tests/framework/generate_manifests.go

@@ -120,9 +120,14 @@ type route struct {
 
 // ScaleObjects contains objects for scale testing.
 type ScaleObjects struct {
-	// BaseObjects contains objects that are common to all scale iterations.
+	// BaseObjects contains objects that should be created first:
+	// secrets and other foundational resources.
 	BaseObjects []client.Object
-	// ScaleIterationGroups contains objects for each scale iteration.
+	// GatewayAndServiceObjects contains backend services, deployments, and Gateway objects.
+	// These are created after BaseObjects to ensure endpoints are ready before traffic.
+	GatewayAndServiceObjects []client.Object
+	// ScaleIterationGroups contains HTTPRoute objects for each scale iteration.
+	// These are applied after GatewayAndServiceObjects to start traffic flow incrementally.
 	ScaleIterationGroups [][]client.Object
 }
 
@@ -150,12 +155,15 @@ func decodeObjects(reader io.Reader) ([]client.Object, error) {
 }
 
 // GenerateScaleListenerObjects generates objects for a given number of listeners for the scale test.
+// Secrets are created first in BaseObjects, then backend services/deployments and Gateway in GatewayAndServiceObjects,
+// and finally HTTPRoutes in ScaleIterationGroups.
 func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, error) {
 	var result ScaleObjects
 
 	listeners := make([]listener, 0)
 	backends := make([]string, 0)
 	secrets := make([]string, 0)
+	routes := make([]route, 0)
 
 	for i := range numListeners {
 		listenerName := fmt.Sprintf("listener-%d", i)
@@ -180,30 +188,34 @@ func GenerateScaleListenerObjects(numListeners int, tls bool) (ScaleObjects, err
 			HostnamePrefix: hostnamePrefix,
 			BackendName:    backendName,
 		}
+		routes = append(routes, r)
 
 		backends = append(backends, backendName)
-
-		objects, err := generateManifests(listeners, []route{r})
-		if err != nil {
-			return ScaleObjects{}, err
-		}
-
-		result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects)
 	}
 
 	secretObjects, err := generateSecrets(secrets)
 	if err != nil {
 		return ScaleObjects{}, err
 	}
-
 	result.BaseObjects = append(result.BaseObjects, secretObjects...)
 
 	backendObjects, err := generateBackendAppObjects(backends)
 	if err != nil {
 		return ScaleObjects{}, err
 	}
+	result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, backendObjects...)
 
-	result.BaseObjects = append(result.BaseObjects, backendObjects...)
+	gatewayObjects, err := generateManifests(listeners, nil)
+	if err != nil {
+		return ScaleObjects{}, err
+	}
+	result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, gatewayObjects...)
+
+	routeObjects, err := generateManifests(nil, routes)
+	if err != nil {
+		return ScaleObjects{}, err
+	}
+	result.ScaleIterationGroups = append(result.ScaleIterationGroups, routeObjects)
 
 	return result, nil
 }
@@ -240,6 +252,21 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) {
 
 	backendName := "backend"
 
+	// Generate backend objects and add to GatewayAndServiceObjects
+	backendObjects, err := generateBackendAppObjects([]string{backendName})
+	if err != nil {
+		return ScaleObjects{}, err
+	}
+	result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, backendObjects...)
+
+	// Generate Gateway object and add to GatewayAndServiceObjects
+	gatewayObjects, err := generateManifests([]listener{l}, nil)
+	if err != nil {
+		return ScaleObjects{}, err
+	}
+	result.GatewayAndServiceObjects = append(result.GatewayAndServiceObjects, gatewayObjects...)
+
+	// Generate HTTPRoute objects for each iteration
 	for i := range numRoutes {
 		r := route{
 			Name:           fmt.Sprintf("route-%d", i),
@@ -248,27 +275,15 @@ func GenerateScaleHTTPRouteObjects(numRoutes int) (ScaleObjects, error) {
 			BackendName:    backendName,
 		}
 
-		var listeners []listener
-		if i == 0 {
-			// only generate a Gateway on the first iteration
-			listeners = []listener{l}
-		}
-
-		objects, err := generateManifests(listeners, []route{r})
+		// Generate only the HTTPRoute (no listeners/gateway)
+		routeObjects, err := generateManifests(nil, []route{r})
 		if err != nil {
 			return ScaleObjects{}, err
 		}
 
-		result.ScaleIterationGroups = append(result.ScaleIterationGroups, objects)
+		result.ScaleIterationGroups = append(result.ScaleIterationGroups, routeObjects)
 	}
 
-	backendObjects, err := generateBackendAppObjects([]string{backendName})
-	if err != nil {
-		return ScaleObjects{}, err
-	}
-
-	result.BaseObjects = backendObjects
-
 	return result, nil
 }
 

tests/framework/results.go

@@ -21,6 +21,12 @@ func CreateResultsDir(testName, version string) (string, error) {
 
 	dirName := filepath.Join(filepath.Dir(pwd), "results", testName, version)
 
+	if _, err := os.Stat(dirName); err == nil {
+		if err := os.RemoveAll(dirName); err != nil {
+			return "", fmt.Errorf("failed to remove existing directory %s: %w", dirName, err)
+		}
+	}
+
 	return dirName, os.MkdirAll(dirName, 0o777)
 }
 

tests/framework/timeout.go

@@ -38,22 +38,26 @@ type TimeoutConfig struct {
 
 	// KubernetesClientTimeout represents the maximum time for Kubernetes client operations.
 	KubernetesClientTimeout time.Duration
+
+	// GatewayListenerUpdateTimeout represents the maximum time for Gateway Listener count to be updated.
+	GatewayListenerUpdateTimeout time.Duration
 }
 
 // DefaultTimeoutConfig populates a TimeoutConfig with the default values.
 func DefaultTimeoutConfig() TimeoutConfig {
 	return TimeoutConfig{
-		CreateTimeout:           60 * time.Second,
-		UpdateTimeout:           60 * time.Second,
-		DeleteTimeout:           10 * time.Second,
-		DeleteNamespaceTimeout:  150 * time.Second,
-		GetTimeout:              10 * time.Second,
-		ManifestFetchTimeout:    10 * time.Second,
-		RequestTimeout:          10 * time.Second,
-		ContainerRestartTimeout: 10 * time.Second,
-		GetLeaderLeaseTimeout:   60 * time.Second,
-		GetStatusTimeout:        60 * time.Second,
-		TestForTrafficTimeout:   60 * time.Second,
-		KubernetesClientTimeout: 10 * time.Second,
+		CreateTimeout:                60 * time.Second,
+		UpdateTimeout:                60 * time.Second,
+		DeleteTimeout:                10 * time.Second,
+		DeleteNamespaceTimeout:       150 * time.Second,
+		GetTimeout:                   10 * time.Second,
+		ManifestFetchTimeout:         10 * time.Second,
+		RequestTimeout:               30 * time.Second,
+		ContainerRestartTimeout:      10 * time.Second,
+		GetLeaderLeaseTimeout:        60 * time.Second,
+		GetStatusTimeout:             60 * time.Second,
+		TestForTrafficTimeout:        60 * time.Second,
+		KubernetesClientTimeout:      10 * time.Second,
+		GatewayListenerUpdateTimeout: 60 * time.Second,
 	}
 }

tests/suite/manifests/scale/httproute.yaml

@@ -0,0 +1,18 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: route
+spec:
+  parentRefs:
+  - name: gateway
+    sectionName: listener
+  hostnames:
+  - "*.example.com"
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /
+    backendRefs:
+    - name: backend
+      port: 80

tests/suite/manifests/scale/upstreams.yaml

@@ -13,25 +13,6 @@ spec:
     port: 80
     protocol: HTTP
 ---
-apiVersion: gateway.networking.k8s.io/v1
-kind: HTTPRoute
-metadata:
-  name: route
-spec:
-  parentRefs:
-  - name: gateway
-    sectionName: listener
-  hostnames:
-  - "*.example.com"
-  rules:
-  - matches:
-    - path:
-        type: PathPrefix
-        value: /
-    backendRefs:
-    - name: backend
-      port: 80
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:

tests/suite/scale_test.go

@@ -40,6 +40,10 @@ var _ = Describe("Scale test", Ordered, Label("nfr", "scale"), func() {
 			"scale/upstreams.yaml",
 		}
 
+		httpRouteManifests = []string{
+			"scale/httproute.yaml",
+		}
+
 		namespace = "scale"
 
 		scrapeInterval = 15 * time.Second
@@ -404,8 +408,12 @@ The logs are attached only if there are errors.
 		Expect(err).ToNot(HaveOccurred())
 		defer ttrCsvFile.Close()
 
+		// Apply BaseObjects first (secrets and other foundational resources)
 		Expect(resourceManager.Apply(objects.BaseObjects)).To(Succeed())
 
+		// Apply GatewayAndServiceObjects next (backend services, deployments, and Gateway)
+		Expect(resourceManager.Apply(objects.GatewayAndServiceObjects)).To(Succeed())
+
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 		defer cancel()
 
@@ -468,6 +476,10 @@ The logs are attached only if there are errors.
 		Expect(resourceManager.ApplyFromFiles(upstreamsManifests, namespace)).To(Succeed())
 		Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed())
 
+		// apply HTTPRoute after upstreams are ready
+		Expect(resourceManager.ApplyFromFiles(httpRouteManifests, namespace)).To(Succeed())
+		Expect(resourceManager.WaitForAppsToBeReady(namespace)).To(Succeed())
+
 		var nginxPodNames []string
 		var err error
 		Eventually(
@@ -512,6 +524,9 @@ The logs are attached only if there are errors.
 		for _, obj := range objects.BaseObjects {
 			obj.SetNamespace(namespace)
 		}
+		for _, obj := range objects.GatewayAndServiceObjects {
+			obj.SetNamespace(namespace)
+		}
 		for _, objs := range objects.ScaleIterationGroups {
 			for _, obj := range objs {
 				obj.SetNamespace(namespace)
@@ -1012,13 +1027,13 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
 			checkGatewayListeners := func(num int) {
 				Eventually(
 					func() error {
-						ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout)
+						ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.GetTimeout*2)
 						defer cancel()
 
 						var gw v1.Gateway
 						key := types.NamespacedName{Namespace: ns.Name, Name: "gateway"}
 						if err := resourceManager.K8sClient.Get(ctx, key, &gw); err != nil {
-							return err
+							return fmt.Errorf("failed to get gateway: %w", err)
 						}
 
 						if len(gw.Status.Listeners) != num {
@@ -1028,8 +1043,8 @@ var _ = Describe("Zero downtime scale test", Ordered, Label("nfr", "zero-downtim
 						return nil
 					},
 				).
-					WithTimeout(5 * time.Second).
-					WithPolling(100 * time.Millisecond).
+					WithTimeout(timeoutConfig.GatewayListenerUpdateTimeout).
+					WithPolling(1 * time.Second).
 					Should(Succeed())
 			}