test(cni): update k8s and calico (#14707)

We were stuck on k8s v1.27 because of k3d-io/k3d#1375, which is no
longer an issue.

This change removes that constraint and installs the latest version of
Calico using the tigera-operator.

Author: alpeb

bin/_test-helpers.sh

@@ -313,11 +313,7 @@ start_test() {
       config=("$name" "${config[@]}" --no-lb --k3s-arg --cluster-domain=custom.domain --k3s-arg '--disable=servicelb,traefik@server:0' --image "$k8s_version_max")
       ;;
     cni-calico-deep)
-      # This requires k8s v1.27.6-k3s1 because after that Calico won't work.
-      # We have to use a config file because that version can't be set via the
-      # --image flag.
-      # See https://github.com/k3d-io/k3d/issues/1375
-      config=("$name" "${config[@]}" --no-lb --k3s-arg --write-kubeconfig-mode=644 --k3s-arg --flannel-backend=none --k3s-arg --cluster-cidr=192.168.0.0/16 --k3s-arg '--disable=servicelb,traefik@server:0' --config "$testdir"/deep/calico-k3d.yml)
+      config=("$name" "${config[@]}" --no-lb --k3s-arg --write-kubeconfig-mode=644 --k3s-arg --flannel-backend=none --k3s-arg --cluster-cidr=192.168.0.0/16 --k3s-arg '--disable=servicelb,traefik@server:0')
       ;;
     multicluster)
       config=("${config[@]}" --network multicluster-test --image "$k8s_version_max")

test/integration/deep/calico-k3d.yml

@@ -27,17 +27,42 @@ func TestInstallCalico(t *testing.T) {
 		return
 	}
 
-	out, err := TestHelper.Kubectl("", []string{"apply", "-f", "https://k3d.io/v5.1.0/usage/advanced/calico.yaml"}...)
-	if err != nil {
-		testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
-			"kubectl apply command failed\n%s", out)
-	}
+	// install Calico as per instructions on
+	// https://k3d.io/v5.8.3/usage/advanced/calico/#1-create-the-cluster-without-flannel
+	// there's a lot of waiting involved here as the various components come up, so the easiest
+	// is to retry the steps until they succeed
+	var out string
+	err := testutil.RetryFor(time.Minute, func() error {
+		var err error
+		out, err = TestHelper.Kubectl("", []string{"apply", "-f", "https://raw.githubusercontent.com/projectcalico/calico/v3.31.0/manifests/tigera-operator.yaml"}...)
+		if err != nil {
+			return err
+		}
+
+		deploys := map[string]testutil.DeploySpec{
+			"tigera-operator": {
+				Namespace: "tigera-operator",
+				Replicas:  1,
+			},
+		}
+		TestHelper.WaitRollout(t, deploys)
 
-	time.Sleep(10 * time.Second)
-	o, err := TestHelper.Kubectl("", "--namespace=kube-system", "wait", "--for=condition=available", "--timeout=120s", "deploy/calico-kube-controllers")
+		out, err = TestHelper.Kubectl("", []string{"apply", "-f", "https://raw.githubusercontent.com/projectcalico/calico/v3.31.0/manifests/custom-resources.yaml"}...)
+		if err != nil {
+			return err
+		}
+
+		for _, system := range []string{"apiserver", "calico", "goldmane", "ippools", "whisker"} {
+			out, err = TestHelper.Kubectl("", "wait", "--for=condition=available", "--timeout=120s", "tigerastatus", system)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
 	if err != nil {
-		testutil.AnnotatedFatalf(t, "failed to wait for condition=available for calico resources",
-			"failed to wait for condition=available for calico resources: %s: %s", err, o)
+		testutil.AnnotatedFatalf(t, "failed to install calico", "failed to install calico: %s: %s", err, out)
 	}
 }