fix: fix the e2e and adjust log (#265)

ryanzhang-oss · web-flow · commit 09ad8bfd4385 · 2025-10-10T11:12:57.000-07:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -57,34 +57,6 @@ jobs:
           ## Comma-separated list of files to upload
           files: ./it-coverage.xml;./ut-coverage.xml
   
-  e2e-tests-v1alpha1:
-    runs-on: ubuntu-latest
-    needs: [
-      detect-noop,
-    ]
-    if: needs.detect-noop.outputs.noop != 'true'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v6
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v5
-      
-      - name: Install Kind
-        # Before updating the kind version to use, verify that the current kind image
-        # is still supported by the version.
-        run: |
-          go install sigs.k8s.io/kind@v0.20.0
-
-      - name: Run e2e tests
-        run: |
-          OUTPUT_TYPE=type=docker make docker-build-member-agent docker-build-hub-agent docker-build-refresh-token e2e-tests-v1alpha1
-        env:
-          KUBECONFIG: '/home/runner/.kube/config'
-          HUB_SERVER_URL: 'https://172.19.0.2:6443'
-  
   e2e-tests:
     strategy:
       fail-fast: false
@@ -130,13 +102,13 @@ jobs:
       
       - name: Install Ginkgo CLI
         run: |
-          go install github.com/onsi/ginkgo/v2/ginkgo@v2.19.1
+          go install github.com/onsi/ginkgo/v2/ginkgo@v2.23.4
       
       - name: Install Kind
         # Before updating the kind version to use, verify that the current kind image
         # is still supported by the version.
         run: |
-          go install sigs.k8s.io/kind@v0.22.0
+          go install sigs.k8s.io/kind@v0.30.0
 
       - name: Run e2e tests
         run: |
diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 REGISTRY ?= ghcr.io
-KIND_IMAGE ?= kindest/node:v1.31.0
+KIND_IMAGE ?= kindest/node:v1.33.4
 ifndef TAG
 	TAG ?= $(shell git rev-parse --short=7 HEAD)
 endif
diff --git a/charts/hub-agent/values.yaml b/charts/hub-agent/values.yaml
@@ -49,5 +49,4 @@ hubAPIQPS: 250
 hubAPIBurst: 1000
 MaxConcurrentClusterPlacement: 100
 ConcurrentResourceChangeSyncs: 20
-logFileMaxSize: "10000000"
 MaxFleetSizeSupported: 100
diff --git a/hack/Azure/setup/createHubCluster.sh b/hack/Azure/setup/createHubCluster.sh
@@ -40,7 +40,7 @@ helm install hub-agent charts/hub-agent/ \
   --set ConcurrentRolloutSyncs=20 \
   --set hubAPIQPS=100 \
   --set hubAPIBurst=1000 \
-  --set logFileMaxSize=100000000 \
+  --set logFileMaxSize=5000 \
   --set MaxFleetSizeSupported=100
 
 # Check the status of the hub agent
diff --git a/pkg/controllers/workapplier/process.go b/pkg/controllers/workapplier/process.go
@@ -229,7 +229,7 @@ func (r *Reconciler) takeOverInMemberClusterObjectIfApplicable(
 ) (shouldSkipProcessing bool) {
 	if !shouldInitiateTakeOverAttempt(bundle.inMemberClusterObj, work.Spec.ApplyStrategy, expectedAppliedWorkOwnerRef) {
 		// Takeover is not necessary; proceed with the processing.
-		klog.V(2).InfoS("Takeover is not needed; skip the step")
+		klog.V(2).InfoS("Takeover is not needed; skip the step", "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", bundle.workResourceIdentifierStr)
 		return false
 	}
 
diff --git a/pkg/propertyprovider/azure/provider_integration_test.go b/pkg/propertyprovider/azure/provider_integration_test.go
@@ -320,11 +320,8 @@ var (
 )
 
 var (
-	// The nodes below use actual capacities of their respective AKS SKUs; for more information,
-	// see:
-	// https://learn.microsoft.com/en-us/azure/virtual-machines/av2-series (for A-series nodes),
-	// https://learn.microsoft.com/en-us/azure/virtual-machines/sizes-b-series-burstable (for B-series nodes), and
-	// https://learn.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series (for D/DS v2 series nodes).
+	// The nodes below use actual capacities of their respective AKS SKUs; We need to change them periodically as
+	// they will be deprecated over time.
 	nodes = []corev1.Node{
 		{
 			ObjectMeta: metav1.ObjectMeta{
diff --git a/pkg/propertyprovider/azure/suite_test.go b/pkg/propertyprovider/azure/suite_test.go
@@ -43,10 +43,10 @@ const (
 
 const (
 	region = "eastus"
-
-	aksNodeSKU1 = "Standard_B4ms"
-	aksNodeSKU2 = "Standard_A4_v2"
-	aksNodeSKU3 = "Standard_F4s"
+	// we need to regularly update the SKUs used in tests as some of them get deprecated over time.
+	aksNodeSKU1 = "Standard_D8s_v5"
+	aksNodeSKU2 = "Standard_E16_v5"
+	aksNodeSKU3 = "Standard_M16ms"
 	// Note (chenyu1): cross-reference between the Azure VM SKU list and the Azure Retail Prices API
 	// for a list of currently known SKUs to be missing from the Azure Retail Prices API.
 	aksNodeKnownMissingSKU1 = "Standard_DS2_v2"
diff --git a/pkg/propertyprovider/azure/trackers/nodes.go b/pkg/propertyprovider/azure/trackers/nodes.go
@@ -328,7 +328,7 @@ func (nt *NodeTracker) trackSKU(node *corev1.Node) bool {
 		return true
 	default:
 		// No further action is needed if the node's SKU remains the same.
-		klog.V(2).InfoS("The node's SKU has not changed", "sku", sku, "node", klog.KObj(node))
+		klog.V(3).InfoS("The node's SKU has not changed", "sku", sku, "node", klog.KObj(node))
 		return false
 	}
 }
diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go
@@ -1506,7 +1506,7 @@ func crpStatusWithExternalStrategyActual(
 		}
 
 		if diff := cmp.Diff(crp.Status, wantStatus, placementStatusCmpOptions...); diff != "" {
-			return fmt.Errorf("CRP status diff (-got, +want): %s", diff)
+			return fmt.Errorf("CRP status diff (-got, +want): %s for CRP %s", diff, crpName)
 		}
 		return nil
 	}
@@ -1529,11 +1529,11 @@ func customizedPlacementStatusUpdatedActual(
 		if wantObservedResourceIndex == "0" {
 			// When the observedResourceIndex is "0", it means the placement is just created and the placement controller
 			if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptionsOnCreate...); diff != "" {
-				return fmt.Errorf("Placement status diff (-got, +want): %s", diff)
+				return fmt.Errorf("Placement status diff (-got, +want): %s for placement %v", diff, placementKey)
 			}
 		} else {
 			if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptions...); diff != "" {
-				return fmt.Errorf("Placement status diff (-got, +want): %s", diff)
+				return fmt.Errorf("Placement status diff (-got, +want): %s for placement %v", diff, placementKey)
 			}
 		}
 		return nil
diff --git a/test/e2e/kindconfigs/cluster-1.yaml b/test/e2e/kindconfigs/cluster-1.yaml
@@ -7,3 +7,5 @@ apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
 - role: worker
+- role: worker
+- role: worker
diff --git a/test/e2e/kindconfigs/cluster-2.yaml b/test/e2e/kindconfigs/cluster-2.yaml
@@ -8,3 +8,4 @@ nodes:
 - role: control-plane
 - role: worker
 - role: worker
+- role: worker
diff --git a/test/e2e/resources/test-daemonset.yaml b/test/e2e/resources/test-daemonset.yaml
@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
         - name: fluentd-elasticsearch
-          image: quay.io/fluentd_elasticsearch/fluentd:v2.5.2
+          image: quay.io/fluentd_elasticsearch/fluentd:v4.11
           resources:
             limits:
               cpu: 250m
diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh
@@ -6,7 +6,7 @@ set -o pipefail
 
 # Before updating the default kind image to use, verify that the version is supported
 # by the current kind release.
-KIND_IMAGE="${KIND_IMAGE:-kindest/node:v1.31.0}"
+KIND_IMAGE="${KIND_IMAGE:-kindest/node:v1.33.4}"
 KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
 MEMBER_CLUSTER_COUNT=$1
 
@@ -126,7 +126,7 @@ helm install hub-agent ../../charts/hub-agent/ \
     --set webhookClientConnectionType=service \
     --set forceDeleteWaitTime="1m0s" \
     --set clusterUnhealthyThreshold="3m0s" \
-    --set logFileMaxSize=1000000 \
+    --set logFileMaxSize=100000 \
     --set resourceSnapshotCreationMinimumInterval=$RESOURCE_SNAPSHOT_CREATION_MINIMUM_INTERVAL \
     --set resourceChangesCollectionDuration=$RESOURCE_CHANGES_COLLECTION_DURATION
 
diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go
@@ -88,7 +88,7 @@ const (
 	// Do not bump this value unless you have a good reason. This is to safeguard any performance related regressions.
 	eventuallyDuration = time.Second * 10
 	// this is for workload related test cases
-	workloadEventuallyDuration = time.Second * 45
+	workloadEventuallyDuration = time.Second * 645
 	// This is for cluster setup.
 	longEventuallyDuration = time.Minute * 2
 	eventuallyInterval     = time.Millisecond * 250
diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go
@@ -330,7 +330,7 @@ func checkIfAzurePropertyProviderIsWorking() {
 				return fmt.Errorf("member cluster status conditions diff (-got, +want):\n%s", diff)
 			}
 			return nil
-		}, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to confirm that Azure property provider is up and running")
+		}, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to confirm that Azure property provider is up and running on cluster", memberCluster.ClusterName)
 	}
 }
 
@@ -739,8 +739,19 @@ func createWorkResources() {
 }
 
 func createNamespace() {
-	ns := appNamespace()
-	Expect(hubClient.Create(ctx, &ns)).To(Succeed(), "Failed to create namespace %s", ns.Name)
+	var ns corev1.Namespace
+	Eventually(func() error {
+		ns = appNamespace()
+		err := hubClient.Create(ctx, &ns)
+		if k8serrors.IsAlreadyExists(err) {
+			err = hubClient.Get(ctx, types.NamespacedName{Name: ns.Name}, &ns)
+			if err != nil {
+				return fmt.Errorf("failed to get the namespace %s, err is %+w", ns.Name, err)
+			}
+			return fmt.Errorf("namespace %s already exists, delete time is %v", ns.Name, ns.GetDeletionTimestamp())
+		}
+		return err
+	}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create namespace %s", ns.Name)
 }
 
 func createConfigMap() {
diff --git a/test/upgrade/setup.sh b/test/upgrade/setup.sh
@@ -73,7 +73,7 @@ helm install hub-agent charts/hub-agent/ \
     --set webhookClientConnectionType=service \
     --set forceDeleteWaitTime="1m0s" \
     --set clusterUnhealthyThreshold="3m0s" \
-    --set logFileMaxSize=1000000
+    --set logFileMaxSize=100000
 
 # Install the member agent and related components to the member clusters.
 
diff --git a/test/upgrade/upgrade.sh b/test/upgrade/upgrade.sh
@@ -62,7 +62,7 @@ if [ -n "$UPGRADE_HUB_SIDE" ]; then
         --set webhookClientConnectionType=service \
         --set forceDeleteWaitTime="1m0s" \
         --set clusterUnhealthyThreshold="3m0s" \
-        --set logFileMaxSize=1000000
+        --set logFileMaxSize=100000
 fi
 
 # Query the URL of the hub cluster API server.

Original file line number	Diff line number	Diff line change
`@@ -229,7 +229,7 @@ func (r *Reconciler) takeOverInMemberClusterObjectIfApplicable(`
`229`	`229`	`) (shouldSkipProcessing bool) {`
`230`	`230`	`if !shouldInitiateTakeOverAttempt(bundle.inMemberClusterObj, work.Spec.ApplyStrategy, expectedAppliedWorkOwnerRef) {`
`231`	`231`	`// Takeover is not necessary; proceed with the processing.`
`232`		`- klog.V(2).InfoS("Takeover is not needed; skip the step")`
	`232`	`+ klog.V(2).InfoS("Takeover is not needed; skip the step", "work", klog.KObj(work), "GVR", *bundle.gvr, "manifestObj", bundle.workResourceIdentifierStr)`
`233`	`233`	`return false`
`234`	`234`	`}`
`235`	`235`
Original file line number	Diff line number	Diff line change
`@@ -320,11 +320,8 @@ var (`
`320`	`320`	`)`
`321`	`321`
`322`	`322`	`var (`
`323`		`- // The nodes below use actual capacities of their respective AKS SKUs; for more information,`
`324`		`- // see:`
`325`		`- // https://learn.microsoft.com/en-us/azure/virtual-machines/av2-series (for A-series nodes),`
`326`		`- // https://learn.microsoft.com/en-us/azure/virtual-machines/sizes-b-series-burstable (for B-series nodes), and`
`327`		`- // https://learn.microsoft.com/en-us/azure/virtual-machines/dv2-dsv2-series (for D/DS v2 series nodes).`
	`323`	`+ // The nodes below use actual capacities of their respective AKS SKUs; We need to change them periodically as`
	`324`	`+ // they will be deprecated over time.`
`328`	`325`	`nodes = []corev1.Node{`
`329`	`326`	`{`
`330`	`327`	`ObjectMeta: metav1.ObjectMeta{`
Original file line number	Diff line number	Diff line change
`@@ -328,7 +328,7 @@ func (nt NodeTracker) trackSKU(node corev1.Node) bool {`
`328`	`328`	`return true`
`329`	`329`	`default:`
`330`	`330`	`// No further action is needed if the node's SKU remains the same.`
`331`		`- klog.V(2).InfoS("The node's SKU has not changed", "sku", sku, "node", klog.KObj(node))`
	`331`	`+ klog.V(3).InfoS("The node's SKU has not changed", "sku", sku, "node", klog.KObj(node))`
`332`	`332`	`return false`
`333`	`333`	`}`
`334`	`334`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1506,7 +1506,7 @@ func crpStatusWithExternalStrategyActual(`
`1506`	`1506`	`}`
`1507`	`1507`
`1508`	`1508`	`if diff := cmp.Diff(crp.Status, wantStatus, placementStatusCmpOptions...); diff != "" {`
`1509`		`- return fmt.Errorf("CRP status diff (-got, +want): %s", diff)`
	`1509`	`+ return fmt.Errorf("CRP status diff (-got, +want): %s for CRP %s", diff, crpName)`
`1510`	`1510`	`}`
`1511`	`1511`	`return nil`
`1512`	`1512`	`}`
`@@ -1529,11 +1529,11 @@ func customizedPlacementStatusUpdatedActual(`
`1529`	`1529`	`if wantObservedResourceIndex == "0" {`
`1530`	`1530`	`// When the observedResourceIndex is "0", it means the placement is just created and the placement controller`
`1531`	`1531`	`if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptionsOnCreate...); diff != "" {`
`1532`		`- return fmt.Errorf("Placement status diff (-got, +want): %s", diff)`
	`1532`	`+ return fmt.Errorf("Placement status diff (-got, +want): %s for placement %v", diff, placementKey)`
`1533`	`1533`	`}`
`1534`	`1534`	`} else {`
`1535`	`1535`	`if diff := cmp.Diff(placement.GetPlacementStatus(), wantStatus, placementStatusCmpOptions...); diff != "" {`
`1536`		`- return fmt.Errorf("Placement status diff (-got, +want): %s", diff)`
	`1536`	`+ return fmt.Errorf("Placement status diff (-got, +want): %s for placement %v", diff, placementKey)`
`1537`	`1537`	`}`
`1538`	`1538`	`}`
`1539`	`1539`	`return nil`