Add retry for SSA requests against Kubernetes < v1.29 in clusterctl

upgrade tests

Signed-off-by: Stefan Büringer [email protected]

Author: sbueringer

test/e2e/clusterctl_upgrade.go

@@ -28,6 +28,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/blang/semver/v4"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	coordinationv1 "k8s.io/api/coordination/v1"
@@ -726,15 +727,28 @@ func ClusterctlUpgradeSpec(ctx context.Context, inputGetter func() ClusterctlUpg
 			verifyV1Beta2ConditionsTrueV1Beta1(ctx, managementClusterProxy.GetClient(), workloadCluster.Name, workloadCluster.Namespace,
 				[]string{clusterv1.AvailableV1Beta2Condition, clusterv1.ReadyV1Beta2Condition})
 
-			Byf("[%d] Verify client-side SSA still works", i)
-			clusterUpdate := &unstructured.Unstructured{}
-			clusterUpdate.SetGroupVersionKind(clusterv1beta1.GroupVersion.WithKind("Cluster"))
-			clusterUpdate.SetNamespace(workloadCluster.Namespace)
-			clusterUpdate.SetName(workloadCluster.Name)
-			clusterUpdate.SetLabels(map[string]string{
-				fmt.Sprintf("test-label-upgrade-%d", i): "test-label-value",
-			})
-			err = managementClusterProxy.GetClient().Patch(ctx, clusterUpdate, client.Apply, client.FieldOwner("e2e-test-client"))
+			// Note: It is a known issue on Kubernetes < v1.29 that SSA sometimes fail:
+			// https://github.com/kubernetes/kubernetes/issues/117356
+			tries := 1
+			initKubernetesVersionParsed, err := semver.ParseTolerant(initKubernetesVersion)
+			Expect(err).ToNot(HaveOccurred())
+			if initKubernetesVersionParsed.LT(semver.MustParse("1.29.0")) {
+				tries = 10
+			}
+			for range tries {
+				Byf("[%d] Verify client-side SSA still works", i)
+				clusterUpdate := &unstructured.Unstructured{}
+				clusterUpdate.SetGroupVersionKind(clusterv1beta1.GroupVersion.WithKind("Cluster"))
+				clusterUpdate.SetNamespace(workloadCluster.Namespace)
+				clusterUpdate.SetName(workloadCluster.Name)
+				clusterUpdate.SetLabels(map[string]string{
+					fmt.Sprintf("test-label-upgrade-%d", i): "test-label-value",
+				})
+				err = managementClusterProxy.GetClient().Patch(ctx, clusterUpdate, client.Apply, client.FieldOwner("e2e-test-client"))
+				if err == nil {
+					break
+				}
+			}
 			Expect(err).ToNot(HaveOccurred())
 
 			Byf("[%d] THE UPGRADED MANAGEMENT CLUSTER WORKS!", i)