test: experimental: various E2E improvements (#276)

Author: michaelawyu

apis/cluster/v1beta1/zz_generated.deepcopy.go

@@ -93,6 +93,11 @@ var (
 	enablePprof                  = flag.Bool("enable-pprof", false, "enable pprof profiling")
 	pprofPort                    = flag.Int("pprof-port", 6065, "port for pprof profiling")
 	hubPprofPort                 = flag.Int("hub-pprof-port", 6066, "port for hub pprof profiling")
+	hubQPS                       = flag.Float64("hub-api-qps", 50, "QPS to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
+	hubBurst                     = flag.Int("hub-api-burst", 500, "Burst to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
+	memberQPS                    = flag.Float64("member-api-qps", 250, "QPS to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
+	memberBurst                  = flag.Int("member-api-burst", 1000, "Burst to use while talking with fleet-apiserver. Doesn't cover events and node heartbeat apis which rate limiting is controlled by a different set of flags.")
+
 	// Work applier requeue rate limiter settings.
 	workApplierRequeueRateLimiterAttemptsWithFixedDelay                              = flag.Int("work-applier-requeue-rate-limiter-attempts-with-fixed-delay", 1, "If set, the work applier will requeue work objects with a fixed delay for the specified number of attempts before switching to exponential backoff.")
 	workApplierRequeueRateLimiterFixedDelaySeconds                                   = flag.Float64("work-applier-requeue-rate-limiter-fixed-delay-seconds", 5.0, "If set, the work applier will requeue work objects with this fixed delay in seconds for the specified number of attempts before switching to exponential backoff.")
@@ -150,6 +155,8 @@ func main() {
 		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
 	}
 	hubConfig, err := buildHubConfig(hubURL, *useCertificateAuth, *tlsClientInsecure)
+	hubConfig.QPS = float32(*hubQPS)
+	hubConfig.Burst = *hubBurst
 	if err != nil {
 		klog.ErrorS(err, "Failed to build Kubernetes client configuration for the hub cluster")
 		klog.FlushAndExit(klog.ExitFlushTimeout, 1)
@@ -164,6 +171,8 @@ func main() {
 	mcNamespace := fmt.Sprintf(utils.NamespaceNameFormat, mcName)
 
 	memberConfig := ctrl.GetConfigOrDie()
+	memberConfig.QPS = float32(*memberQPS)
+	memberConfig.Burst = *memberBurst
 	// we place the leader election lease on the member cluster to avoid adding load to the hub
 	hubOpts := ctrl.Options{
 		Scheme: scheme,

apis/placement/v1alpha1/zz_generated.deepcopy.go

@@ -7,5 +7,3 @@ apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
 - role: worker
-- role: worker
-- role: worker

apis/placement/v1beta1/zz_generated.deepcopy.go

@@ -8,4 +8,3 @@ nodes:
 - role: control-plane
 - role: worker
 - role: worker
-- role: worker

apis/v1alpha1/zz_generated.deepcopy.go

@@ -277,15 +277,12 @@ var _ = Describe("mixed ClusterResourcePlacement and ResourcePlacement negative
 			Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName)
 
 			By("creating a CRP that uses PickAll strategy")
+			// Note that this CRP will take over namespaces.
 			createCRP(crpName)
 		})
 
 		AfterAll(func() {
 			ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: workNamespaceName}, allMemberClusters)
-			By("CRP should take over the work resources")
-			// ConfigMap on the hub was deleted in the previous step.
-			crpStatusUpdatedActual := crpStatusUpdatedActual(workNamespaceIdentifiers(), allMemberClusterNames, nil, "1")
-			Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
 			ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters)
 		})
 
@@ -345,5 +342,34 @@ var _ = Describe("mixed ClusterResourcePlacement and ResourcePlacement negative
 			}
 			Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName)
 		})
+
+		// The CRP must give up on the configMap first, otherwise it might attempt re-create the configMap
+		// during the cleanup process.
+		It("can update the CRP to give up on the configMap", func() {
+			Eventually(func() error {
+				crp := &placementv1beta1.ClusterResourcePlacement{}
+				if err := hubClient.Get(ctx, types.NamespacedName{Name: crpName}, crp); err != nil {
+					return fmt.Errorf("failed to get the CRP %s: %w", crpName, err)
+				}
+
+				crp.Spec.ResourceSelectors = namespaceOnlySelector()
+				if err := hubClient.Update(ctx, crp); err != nil {
+					return fmt.Errorf("failed to update the CRP %s: %w", crpName, err)
+				}
+				return nil
+			}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update the CRP %s to give up on the configMap", crpName)
+		})
+
+		It("should update CRP status as expected", func() {
+			nsOnlyResIds := []placementv1beta1.ResourceIdentifier{
+				{
+					Kind:    "Namespace",
+					Name:    workNamespaceName,
+					Version: "v1",
+				},
+			}
+			crpStatusUpdatedActual := crpStatusUpdatedActual(nsOnlyResIds, allMemberClusterNames, nil, "1")
+			Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
+		})
 	})
 })

cmd/memberagent/main.go

@@ -25,6 +25,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -601,9 +602,30 @@ var _ = Describe("validating CRP when resources exists", Ordered, func() {
 		})
 
 		AfterAll(func() {
-			ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters)
+			// Must delete the conflicting CRP first, otherwise it might re-create the resources when we check
+			// if the original CRP has been fully deleted.
+			//
+			// And here the test suite will not use the shared common deletion logic as it will attempt to verify
+			// resources that are not managed by the conflicting CRP.
+			Eventually(func() error {
+				conflictedCRP := &placementv1beta1.ClusterResourcePlacement{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: conflictedCRPName,
+					},
+				}
+
+				if err := hubClient.Delete(ctx, conflictedCRP); err != nil && !errors.IsNotFound(err) {
+					return fmt.Errorf("failed to delete CRP %s: %w", conflictedCRPName, err)
+				}
 
-			ensureCRPAndRelatedResourcesDeleted(conflictedCRPName, allMemberClusters)
+				// Wait until the CRP is fully deleted.
+				if err := hubClient.Get(ctx, types.NamespacedName{Name: conflictedCRPName}, conflictedCRP); !errors.IsNotFound(err) {
+					return fmt.Errorf("CRP %s is still present or an unexpected error has occurred: %w", conflictedCRPName, err)
+				}
+				return nil
+			}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to delete CRP %s", conflictedCRPName)
+
+			ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters)
 		})
 	})
 })