Merge pull request #29428 from tssurya/fix-eventually-timeout-seconds-for-udn-tests

OCPBUGS-48238,OCPBUGS-48449,OCPBUGS-48450: Set the poll timeout and poll interval in eventually directly for the tests

Author: openshift-merge-bot[bot]

test/extended/networking/network_segmentation.go

@@ -38,6 +38,25 @@ import (
 
 const openDefaultPortsAnnotation = "k8s.ovn.org/open-default-ports"
 
+// NOTE: We are observing pod creation requests taking more than two minutes t
+// reach the CNI for the CNI to do the necessary plumbing. This is causing tests
+// to timeout since pod doesn't go into ready state.
+// See https://issues.redhat.com/browse/OCPBUGS-48362 for details. We can revisit
+// these values when that bug is fixed but given the Kubernetes test default for a
+// pod to startup is 5mins: https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/test/e2e/framework/timeouts.go#L22-L23
+// we are not too far from the mark or against test policy
+const podReadyPollTimeout = 4 * time.Minute
+const podReadyPollInterval = 6 * time.Second
+
+// NOTE: Upstream, we use either the default of gomega which is 1sec polltimeout with 10ms pollinterval OR
+// the tests have hardcoded values with 5sec being common for polltimeout and 10ms for pollinterval
+// This is being changed to be 10seconds poll timeout to account for infrastructure complexity between
+// OpenShift and KIND clusters. Also changing the polling interval to be 1 second so that in both
+// Eventually and Consistently blocks we get at least 10 retries (10/1) in good conditions and 5 retries (10/2) in
+// bad conditions since connectToServer util has a 2 second timeout.
+const serverConnectPollTimeout = 10 * time.Second
+const serverConnectPollInterval = 1 * time.Second
+
 var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks]", func() {
 	// TODO: so far, only the isolation tests actually require this PSA ... Feels wrong to run everything priviliged.
 	// I've tried to have multiple kubeframeworks (from multiple OCs) running (with different project names) but
@@ -263,7 +282,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 							By("checking the default network pod can't reach UDN pod on IP " + destIP)
 							Consistently(func() bool {
 								return connectToServer(podConfiguration{namespace: defaultPod.Namespace, name: defaultPod.Name}, destIP, port) != nil
-							}, 5*time.Second).Should(BeTrue())
+							}, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue())
 						}
 
 						defaultIPv4, defaultIPv6, err := podIPsForDefaultNetwork(
@@ -280,11 +299,11 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 							By("checking the default network client pod can reach default pod on IP " + destIP)
 							Eventually(func() bool {
 								return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, defaultPort) == nil
-							}).Should(BeTrue())
+							}, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue())
 							By("checking the UDN pod can't reach the default network pod on IP " + destIP)
 							Consistently(func() bool {
 								return connectToServer(udnPodConfig, destIP, defaultPort) != nil
-							}, 5*time.Second).Should(BeTrue())
+							}, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue())
 						}
 
 						// connectivity check is run every second + 1sec initialDelay
@@ -314,7 +333,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 									ping, "-I", "eth0", "-c", "1", "-W", "1", hostIP.IP,
 								)
 								return err == nil
-							}, 4*time.Second).Should(BeFalse())
+							}, 4*time.Second, 1*time.Second).Should(BeFalse())
 						}
 
 						By("asserting UDN pod can reach the kapi service in the default network")
@@ -331,7 +350,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 								"--insecure",
 								"https://kubernetes.default/healthz")
 							return err == nil
-						}, 5*time.Second).Should(BeTrue())
+						}, 5*time.Second, 1*time.Second).Should(BeTrue())
 
 						By("asserting UDN pod can't reach default services via default network interface")
 						// route setup is already done, get kapi IPs
@@ -353,7 +372,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 									"--insecure",
 									fmt.Sprintf("https://%s/healthz", kapiIP))
 								return err != nil
-							}, 5*time.Second).Should(BeTrue())
+							}, 5*time.Second, 1*time.Second).Should(BeTrue())
 						}
 					},
 					Entry(
@@ -662,7 +681,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 						_ = nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Delete(ctx, testUdnName, metav1.DeleteOptions{})
 						_, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Get(ctx, testUdnName, metav1.GetOptions{})
 						return err
-					}).ShouldNot(HaveOccurred(),
+					}, udnInUseDeleteTimeout, deleteNetworkInterval).ShouldNot(HaveOccurred(),
 						"should fail to delete UserDefinedNetwork associated NetworkAttachmentDefinition when used")
 
 					By("verify UserDefinedNetwork status reports consuming pod")
@@ -907,7 +926,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 						_ = nadClient.NetworkAttachmentDefinitions(inUseNetTestTenantNamespace).Delete(ctx, testClusterUdnName, metav1.DeleteOptions{})
 						_, err := nadClient.NetworkAttachmentDefinitions(inUseNetTestTenantNamespace).Get(ctx, testClusterUdnName, metav1.GetOptions{})
 						return err
-					}).ShouldNot(HaveOccurred(),
+					}, udnInUseDeleteTimeout, deleteNetworkInterval).ShouldNot(HaveOccurred(),
 						"should fail to delete UserDefinedNetwork associated NetworkAttachmentDefinition when used")
 
 					By("verify CR status reports consuming pod")
@@ -969,6 +988,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 			By("create primary Cluster UDN CR")
 			cudnName := randomNetworkMetaName()
 			cleanup, err := createManifest(f.Namespace.Name, newPrimaryClusterUDNManifest(cudnName, testTenantNamespaces...))
+			Expect(err).NotTo(HaveOccurred())
 			DeferCleanup(func() {
 				cleanup()
 				_, err := e2ekubectl.RunKubectl("", "delete", "clusteruserdefinednetwork", cudnName, "--wait", fmt.Sprintf("--timeout=%ds", 60))
@@ -1060,7 +1080,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 					By("checking the default network pod can't reach UDN pod on IP " + destIP)
 					Consistently(func() bool {
 						return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) != nil
-					}, 5*time.Second).Should(BeTrue())
+					}, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue())
 				}
 
 				By("Open UDN pod port")
@@ -1078,7 +1098,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 					By("checking the default network pod can reach UDN pod on IP " + destIP)
 					Eventually(func() bool {
 						return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) == nil
-					}, 5*time.Second).Should(BeTrue())
+					}, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue())
 				}
 
 				By("Update UDN pod port with the wrong syntax")
@@ -1097,10 +1117,11 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User
 					By("checking the default network pod can't reach UDN pod on IP " + destIP)
 					Eventually(func() bool {
 						return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) != nil
-					}, 5*time.Second).Should(BeTrue())
+					}, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue())
 				}
 				By("Verify syntax error is reported via event")
 				events, err := cs.CoreV1().Events(udnPod.Namespace).List(context.Background(), metav1.ListOptions{})
+				Expect(err).NotTo(HaveOccurred())
 				found := false
 				for _, event := range events.Items {
 					if event.Reason == "ErrorUpdatingResource" && strings.Contains(event.Message, "invalid protocol ppp") {
@@ -1586,7 +1607,7 @@ func runUDNPod(cs clientset.Interface, namespace string, podConfig podConfigurat
 			return v1.PodFailed
 		}
 		return updatedPod.Status.Phase
-	}, 2*time.Minute, 6*time.Second).Should(Equal(v1.PodRunning))
+	}, podReadyPollTimeout, podReadyPollInterval).Should(Equal(v1.PodRunning))
 	return updatedPod
 }
 

test/extended/prometheus/collection_profiles.go

@@ -33,6 +33,8 @@ const (
 	operatorName              = "cluster-monitoring-operator"
 	operatorNamespaceName     = "openshift-monitoring"
 	operatorConfigurationName = "cluster-monitoring-config"
+	pollTimeout               = 15 * time.Minute
+	pollInterval              = 5 * time.Second
 )
 
 var (
@@ -58,9 +60,6 @@ type runner struct {
 var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfiles] The collection profiles feature-set", g.Ordered, func() {
 	defer g.GinkgoRecover()
 
-	o.SetDefaultEventuallyTimeout(15 * time.Minute)
-	o.SetDefaultEventuallyPollingInterval(5 * time.Second)
-
 	r := &runner{}
 
 	g.BeforeAll(func() {
@@ -92,7 +91,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 			}
 
 			return nil
-		}).Should(o.BeNil())
+		}, pollTimeout, pollInterval).Should(o.BeNil())
 		r.originalOperatorConfiguration = operatorConfiguration
 	})
 
@@ -126,7 +125,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 				}
 
 				return nil
-			}).Should(o.BeNil())
+			}, pollTimeout, pollInterval).Should(o.BeNil())
 		})
 
 		g.It("should expose default metrics", func() {
@@ -142,7 +141,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 				}
 
 				return nil
-			}).Should(o.BeNil())
+			}, pollTimeout, pollInterval).Should(o.BeNil())
 		})
 	})
 
@@ -163,7 +162,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 					}
 
 					return nil
-				}).Should(o.BeNil())
+				}, pollTimeout, pollInterval).Should(o.BeNil())
 			}
 		})
 		g.It("should have at least one implementation for each collection profile", func() {
@@ -181,7 +180,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 					}
 
 					return nil
-				}).Should(o.BeNil())
+				}, pollTimeout, pollInterval).Should(o.BeNil())
 			}
 		})
 		g.It("should revert to default collection profile when an empty collection profile value is specified", func() {
@@ -198,7 +197,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 				}
 
 				return nil
-			}).Should(o.BeNil())
+			}, pollTimeout, pollInterval).Should(o.BeNil())
 		})
 	})
 
@@ -218,7 +217,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 				}
 
 				return nil
-			}).Should(o.BeNil())
+			}, pollTimeout, pollInterval).Should(o.BeNil())
 		})
 
 		g.It("should hide default metrics", func() {
@@ -240,7 +239,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 				kubeStateMetricsMonitor = monitors.Items[0]
 
 				return nil
-			}).Should(o.BeNil())
+			}, pollTimeout, pollInterval).Should(o.BeNil())
 
 			var kubeStateMetricsMainMetrics []string
 			kubeStateMetricsMonitorSpec := kubeStateMetricsMonitor.Spec
@@ -297,7 +296,7 @@ var _ = g.Describe("[sig-instrumentation][OCPFeatureGate:MetricsCollectionProfil
 				}
 
 				return nil
-			}).Should(o.BeNil())
+			}, pollTimeout, pollInterval).Should(o.BeNil())
 		})
 	})
 })

test/extended/storage/driver_configuration.go

@@ -26,6 +26,8 @@ import (
 const (
 	projectName  = "csi-driver-configuration"
 	providerName = "csi.vsphere.vmware.com"
+	pollTimeout  = 5 * time.Minute
+	pollInterval = 5 * time.Second
 )
 
 // This is [Serial] because it modifies ClusterCSIDriver.
@@ -37,9 +39,6 @@ var _ = g.Describe("[sig-storage][FeatureGate:VSphereDriverConfiguration][Serial
 		originalDriverConfigSpec *opv1.CSIDriverConfigSpec
 	)
 
-	o.SetDefaultEventuallyTimeout(5 * time.Minute)
-	o.SetDefaultEventuallyPollingInterval(5 * time.Second)
-
 	g.BeforeEach(func() {
 		if !framework.ProviderIs("vsphere") {
 			g.Skip("this test is only expected to work with vSphere clusters")
@@ -143,7 +142,7 @@ var _ = g.Describe("[sig-storage][FeatureGate:VSphereDriverConfiguration][Serial
 				setClusterCSIDriverSnapshotOptions(ctx, oc, t.clusterCSIDriverOptions)
 				o.Eventually(func() error {
 					return loadAndCheckCloudConf(ctx, oc, "Snapshot", t.cloudConfigOptions, t.clusterCSIDriverOptions)
-				}).Should(o.Succeed())
+				}, pollTimeout, pollInterval).Should(o.Succeed())
 
 				validateSnapshotCreation(ctx, oc, t.successfulSnapshotsCreated)
 			})
@@ -231,7 +230,7 @@ func validateSnapshotCreation(ctx context.Context, oc *exutil.CLI, successfulSna
 		pvc, err := oc.AdminKubeClient().CoreV1().PersistentVolumeClaims(oc.Namespace()).Get(ctx, "test-pvc", metav1.GetOptions{})
 		o.Expect(err).NotTo(o.HaveOccurred())
 		return pvc.Status.Phase
-	}).Should(o.Equal(v1.ClaimBound))
+	}, pollTimeout, pollInterval).Should(o.Equal(v1.ClaimBound))
 
 	var wg sync.WaitGroup
 	var snapshotsCreated = make([]string, 0, successfulSnapshotsCreated)
@@ -269,7 +268,7 @@ func validateSnapshotCreation(ctx context.Context, oc *exutil.CLI, successfulSna
 		}
 		e2e.Logf("Snapshots ready: %d/%d", snapshotsReady, successfulSnapshotsCreated)
 		return snapshotsReady
-	}).Should(o.Equal(successfulSnapshotsCreated), "not all snapshots are ready")
+	}, pollTimeout, pollInterval).Should(o.Equal(successfulSnapshotsCreated), "not all snapshots are ready")
 
 	// Next snapshot creation should be over the set limit and fail.
 	failedSnapshotName := "test-snapshot-failed"
@@ -289,7 +288,7 @@ func validateSnapshotCreation(ctx context.Context, oc *exutil.CLI, successfulSna
 		}
 		e2e.Logf("Error validation successful - snapshot: %s readyToUse: %t, error message: %s", failedSnapshotName, ready, errMsg)
 		return strings.Contains(errMsg, "reaches the configured maximum") && !ready
-	}).Should(o.BeTrue(), "snapshot creation should fail")
+	}, pollTimeout, pollInterval).Should(o.BeTrue(), "snapshot creation should fail")
 }
 
 func createTestPod(ctx context.Context, oc *exutil.CLI, pvcName string, namespace string) (*v1.Pod, error) {

test/extended/storage/s3_configuration.go

@@ -2,7 +2,6 @@ package storage
 
 import (
 	"context"
-	"time"
 
 	g "github.com/onsi/ginkgo/v2"
 	o "github.com/onsi/gomega"
@@ -26,9 +25,6 @@ var _ = g.Describe("[sig-imageregistry][OCPFeatureGate:ChunkSizeMiB][Serial][api
 		originalConfigSpec *imageregistryv1.ImageRegistryConfigStorageS3
 	)
 
-	o.SetDefaultEventuallyTimeout(5 * time.Minute)
-	o.SetDefaultEventuallyPollingInterval(5 * time.Second)
-
 	g.BeforeEach(func() {
 
 		skipIfNotS3Storage(oc)