test: increase time waiting for resource cleanup Signed-off-by: Wantong Jiang <[email protected]>

Author: jwtty

test/e2e/actuals_test.go

@@ -18,6 +18,8 @@ package e2e
 
 import (
 	"fmt"
+	"strings"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
@@ -982,12 +984,39 @@ func safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResourceIdentifiers [
 }
 
 func workNamespaceRemovedFromClusterActual(cluster *framework.Cluster) func() error {
-	client := cluster.KubeClient
-
 	ns := appNamespace()
 	return func() error {
-		if err := client.Get(ctx, types.NamespacedName{Name: ns.Name}, &corev1.Namespace{}); !errors.IsNotFound(err) {
-			return fmt.Errorf("work namespace %s still exists or an unexpected error occurred: %w", ns.Name, err)
+		if err := cluster.KubeClient.Get(ctx, types.NamespacedName{Name: ns.Name}, &ns); !errors.IsNotFound(err) {
+			if err == nil {
+				// List all pods and configmaps in the namespace
+				podList := &corev1.PodList{}
+				if listErr := cluster.KubeClient.List(ctx, podList, client.InNamespace(ns.Name)); listErr != nil {
+					return fmt.Errorf("work namespace %s still exists on cluster %s but failed to list pods: %w, ns deletion timestamp: %v, current timestamp: %v",
+						ns.Name, cluster.ClusterName, listErr, ns.GetDeletionTimestamp(), time.Now())
+				}
+				configMapList := &corev1.ConfigMapList{}
+				if listErr := cluster.KubeClient.List(ctx, configMapList, client.InNamespace(ns.Name)); listErr != nil {
+					return fmt.Errorf("work namespace %s still exists on cluster %s but failed to list configmaps: %w, ns deletion timestamp: %v, current timestamp: %v",
+						ns.Name, cluster.ClusterName, listErr, ns.GetDeletionTimestamp(), time.Now())
+				}
+
+				// Build resource status summary
+				var resourceStatus strings.Builder
+				resourceStatus.WriteString(fmt.Sprintf("\nList pods and configmaps in namespace %s:\n", ns.Name))
+				for _, item := range podList.Items {
+					resourceStatus.WriteString(fmt.Sprintf("- Pod: %s/%s\n", item.GetNamespace(), item.GetName()))
+					resourceStatus.WriteString(fmt.Sprintf("  DeletionTimestamp: %v\n", item.GetDeletionTimestamp()))
+					resourceStatus.WriteString(fmt.Sprintf("  Finalizers: %v\n", item.GetFinalizers()))
+				}
+				for _, item := range configMapList.Items {
+					resourceStatus.WriteString(fmt.Sprintf("- ConfigMap: %s/%s\n", item.GetNamespace(), item.GetName()))
+					resourceStatus.WriteString(fmt.Sprintf("  DeletionTimestamp: %v\n", item.GetDeletionTimestamp()))
+					resourceStatus.WriteString(fmt.Sprintf("  Finalizers: %v\n", item.GetFinalizers()))
+				}
+				return fmt.Errorf("work namespace %s still exists on cluster %s: deletion timestamp: %v, current timestamp: %v, resources: %s",
+					ns.Name, cluster.ClusterName, ns.GetDeletionTimestamp(), time.Now(), resourceStatus.String())
+			}
+			return fmt.Errorf("getting work namespace %s failed: %w", ns.Name, err)
 		}
 		return nil
 	}

test/e2e/placement_eviction_test.go

@@ -159,10 +159,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding, taint clus
 	})
 
 	It("should ensure no resources exist on evicted member cluster with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should update cluster resource placement status as expected", func() {
@@ -583,10 +580,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickAll C
 	})
 
 	It("should ensure no resources exist on evicted member cluster with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should update cluster resource placement status as expected", func() {
@@ -776,10 +770,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP
 	})
 
 	It("should ensure no resources exist on evicted member cluster with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should update cluster resource placement status as expected", func() {
@@ -885,10 +876,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP
 	})
 
 	It("should ensure no resources exist on evicted member cluster with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should update cluster resource placement status as expected", func() {
@@ -1170,10 +1158,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP
 	})
 
 	It("should ensure no resources exist on evicted member cluster with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should update cluster resource placement status as expected", func() {
@@ -1279,10 +1264,7 @@ var _ = Describe("ClusterResourcePlacement eviction of bound binding - PickN CRP
 	})
 
 	It("should ensure no resources exist on evicted member cluster with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should update cluster resource placement status as expected", func() {

test/e2e/placement_pickall_test.go

@@ -293,8 +293,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func
 		})
 
 		It("should remove resources on previously matched clusters", func() {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster1EastProd)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources on previously matched clusters")
+			checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster1EastProd})
 		})
 
 		AfterAll(func() {
@@ -918,8 +917,7 @@ var _ = Describe("placing resources using a CRP of PickAll placement type", func
 		})
 
 		It("should remove resources on previously matched clusters", func() {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster2EastCanary)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources on previously matched clusters")
+			checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster2EastCanary})
 		})
 
 		AfterAll(func() {

test/e2e/placement_pickfixed_test.go

@@ -137,8 +137,7 @@ var _ = Describe("placing resources using a CRP of PickFixed placement type", fu
 		})
 
 		It("should remove resources from previously specified clusters", func() {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster1EastProd)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from previously specified clusters")
+			checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster1EastProd})
 		})
 
 		AfterAll(func() {

test/e2e/placement_pickn_test.go

@@ -145,8 +145,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() {
 		})
 
 		It("should remove resources from the downscaled clusters", func() {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster2EastCanary)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from the downscaled clusters")
+			checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster2EastCanary})
 		})
 
 		AfterAll(func() {
@@ -411,8 +410,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() {
 		})
 
 		It("should remove resources from the unpicked clusters", func() {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster2EastCanary)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from the downscaled clusters")
+			checkIfRemovedWorkResourcesFromMemberClusters([]*framework.Cluster{memberCluster2EastCanary})
 		})
 
 		AfterAll(func() {
@@ -548,10 +546,7 @@ var _ = Describe("placing resources using a CRP of PickN placement", func() {
 
 		It("should remove resources from the downscaled clusters", func() {
 			downscaledClusters := []*framework.Cluster{memberCluster3WestProd, memberCluster2EastCanary}
-			for _, cluster := range downscaledClusters {
-				resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-				Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove resources from the downscaled clusters")
-			}
+			checkIfRemovedWorkResourcesFromMemberClusters(downscaledClusters)
 		})
 
 		It("should update CRP status as expected", func() {

test/e2e/taint_toleration_test.go

@@ -114,10 +114,7 @@ var _ = Describe("placing resources using a cluster resource placement with no p
 	})
 
 	It("should ensure no resources exist on member clusters with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should place resources on the selected cluster without taint", func() {
@@ -190,10 +187,7 @@ var _ = Describe("placing resources using a cluster resource placement with no p
 	})
 
 	It("should ensure no resources exist on member clusters with taint", func() {
-		for _, cluster := range taintClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(taintClusters)
 	})
 
 	It("should place resources on the selected cluster without taint", func() {
@@ -307,10 +301,7 @@ var _ = Describe("picking N clusters with affinities and topology spread constra
 	})
 
 	It("should ensure no resources exist on member clusters with untolerated taint", func() {
-		for _, cluster := range unSelectedClusters {
-			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
-			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
-		}
+		checkIfRemovedWorkResourcesFromMemberClusters(unSelectedClusters)
 	})
 
 	AfterAll(func() {

test/e2e/utils_test.go

@@ -754,7 +754,7 @@ func checkIfRemovedWorkResourcesFromMemberClusters(clusters []*framework.Cluster
 		memberCluster := clusters[idx]
 
 		workResourcesRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster)
-		Eventually(workResourcesRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName)
+		Eventually(workResourcesRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName)
 	}
 }