feat: enable Cilium kube-proxy replacement for new clusters

charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml

@@ -33,3 +33,7 @@ socketLB:
 envoy:
   image:
     useDigest: false
+{{- if .EnableKubeProxyReplacement }}
+kubeProxyReplacement: true
+k8sServiceHost: auto
+{{- end }}

common/pkg/capi/utils/anootations.go

@@ -0,0 +1,18 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package utils
+
+import (
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+)
+
+// SkipKubeProxy returns true if the cluster should skip kube proxy installation.
+func SkipKubeProxy(cluster *clusterv1.Cluster) bool {
+	if cluster.Spec.Topology != nil {
+		_, isSkipKubeProxy := cluster.Spec.Topology.ControlPlane.Metadata.Annotations[controlplanev1.SkipKubeProxyAnnotation]
+		return isSkipKubeProxy
+	}
+	return false
+}

examples/capi-quick-start/aws-cluster-cilium-crs.yaml

@@ -16,6 +16,9 @@ spec:
   topology:
     class: aws-quick-start
     controlPlane:
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig

examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml

@@ -16,6 +16,9 @@ spec:
   topology:
     class: aws-quick-start
     controlPlane:
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig

examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml

@@ -57,7 +57,9 @@ spec:
   topology:
     class: nutanix-quick-start
     controlPlane:
-      metadata: {}
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig

examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml

@@ -57,7 +57,9 @@ spec:
   topology:
     class: nutanix-quick-start
     controlPlane:
-      metadata: {}
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig

hack/addons/kustomize/cilium/kustomization.yaml.tmpl

@@ -19,7 +19,7 @@ helmCharts:
   skipTests: true
   namespace: kube-system
   kubeVersion: ${E2E_KUBERNETES_VERSION}
-  valuesFile: ../../../../charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml
+  valuesFile: helm-values.yaml
   # The CRS manifests are generated from the Cilium Helm chart using Kustomize. The Cilium
   # Helm chart uses a Helm hook to generate TLS certificates for Hubble. As the
   # CRS manifests are static those Helm hooks don't apply and so for now Hubble is

hack/addons/update-cilium-manifests.sh

@@ -21,13 +21,18 @@ readonly FILE_NAME="cilium.yaml"
 
 readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/cilium"
 mkdir -p "${ASSETS_DIR}/cilium"
-envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${KUSTOMIZE_BASE_DIR}/kustomization.yaml"
-trap_add "rm -f ${KUSTOMIZE_BASE_DIR}/kustomization.yaml" EXIT
+envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/kustomization.yaml"
+
+cat <<EOF >"${ASSETS_DIR}/gomplate-context.yaml"
+EnableKubeProxyReplacement: true
+EOF
+gomplate -f "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml" \
+  --context .="${ASSETS_DIR}/gomplate-context.yaml" \
+  >"${ASSETS_DIR}/helm-values.yaml"
 
 kustomize build \
   --load-restrictor LoadRestrictionsNone \
-  --enable-helm "${KUSTOMIZE_BASE_DIR}/" >"${ASSETS_DIR}/${FILE_NAME}"
-trap_add "rm -rf ${KUSTOMIZE_BASE_DIR}/charts/" EXIT
+  --enable-helm "${ASSETS_DIR}/" >"${ASSETS_DIR}/${FILE_NAME}"
 
 # The operator manifest in YAML format is pretty big. It turns out that much of that is whitespace. Converting the
 # manifest to JSON without indentation allows us to remove most of the whitespace, reducing the size by more than half.

hack/examples/overlays/clusters/aws/cilium/crs/kustomization.yaml.tmpl

@@ -20,3 +20,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/aws/crs-strategy.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/skip-kube-proxy.yaml

hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl

@@ -14,3 +14,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/cilium.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/skip-kube-proxy.yaml

hack/examples/overlays/clusters/nutanix/cilium/crs/kustomization.yaml.tmpl

@@ -17,3 +17,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/crs-strategy.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/skip-kube-proxy.yaml

hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl

@@ -14,3 +14,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/cilium.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/skip-kube-proxy.yaml

hack/examples/patches/skip-kube-proxy.yaml

@@ -0,0 +1,13 @@
+# Copyright 2025 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: not-used
+spec:
+  topology:
+    controlPlane:
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""

hack/tools/fetch-images/main.go

@@ -259,7 +259,25 @@ func getValuesFileForChartIfNeeded(chartName, carenChartDirectory string) (strin
 	case "snapshot-controller":
 		return filepath.Join(carenChartDirectory, "addons", "csi", "snapshot-controller", defaultHelmAddonFilename), nil
 	case "cilium":
-		return filepath.Join(carenChartDirectory, "addons", "cni", "cilium", defaultHelmAddonFilename), nil
+		f := filepath.Join(carenChartDirectory, "addons", "cni", "cilium", defaultHelmAddonFilename)
+		tempFile, err := os.CreateTemp("", "")
+		if err != nil {
+			return "", fmt.Errorf("failed to create temp file: %w", err)
+		}
+
+		type input struct {
+			EnableKubeProxyReplacement bool
+		}
+		templateInput := input{
+			EnableKubeProxyReplacement: true,
+		}
+
+		err = template.Must(template.New(defaultHelmAddonFilename).ParseFiles(f)).Execute(tempFile, &templateInput)
+		if err != nil {
+			return "", fmt.Errorf("failed to execute helm values template %w", err)
+		}
+
+		return tempFile.Name(), nil
 	// Calico values differ slightly per provider, but that does not have a material imapct on the images required
 	// so we can use the default values file for AWS provider.
 	case "tigera-operator":

pkg/handlers/generic/lifecycle/cni/cilium/handler.go

@@ -221,7 +221,8 @@ func (c *CiliumCNI) apply(
 			),
 			c.client,
 			helmChart,
-		)
+		).
+			WithValueTemplater(templateValues)
 	case "":
 		resp.SetStatus(runtimehooksv1.ResponseStatusFailure)
 		resp.SetMessage("strategy not specified for Cilium CNI addon")

pkg/handlers/generic/lifecycle/cni/cilium/template.go

@@ -0,0 +1,42 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package cilium
+
+import (
+	"bytes"
+	"fmt"
+	"text/template"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+
+	capiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils"
+)
+
+// templateValues enables kube-proxy replacement when skip kube-proxy annotation is set.
+func templateValues(cluster *clusterv1.Cluster, text string) (string, error) {
+	ciliumTemplate, err := template.New("").Parse(text)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse template: %w", err)
+	}
+
+	type input struct {
+		EnableKubeProxyReplacement bool
+	}
+
+	// Assume when kube-proxy is skipped, we should enable Cilium's kube-proxy replacement feature.
+	templateInput := input{
+		EnableKubeProxyReplacement: capiutils.SkipKubeProxy(cluster),
+	}
+
+	var b bytes.Buffer
+	err = ciliumTemplate.Execute(&b, templateInput)
+	if err != nil {
+		return "", fmt.Errorf(
+			"failed setting target Cluster name and namespace in template: %w",
+			err,
+		)
+	}
+
+	return b.String(), nil
+}

pkg/handlers/generic/mutation/kubeproxymode/inject.go

@@ -24,6 +24,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
+	capiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils"
 )
 
 const (
@@ -87,11 +88,6 @@ func (h *kubeProxyMode) Mutate(
 		return fmt.Errorf("failed to get cluster for kube proxy mode mutation: %w", err)
 	}
 
-	isSkipProxy := false
-	if cluster.Spec.Topology != nil {
-		_, isSkipProxy = cluster.Spec.Topology.ControlPlane.Metadata.Annotations[controlplanev1.SkipKubeProxyAnnotation]
-	}
-
 	kubeProxyMode, err := variables.Get[v1alpha1.KubeProxyMode](
 		vars,
 		h.variableName,
@@ -110,6 +106,8 @@ func (h *kubeProxyMode) Mutate(
 		kubeProxyMode,
 	)
 
+	isSkipProxy := capiutils.SkipKubeProxy(cluster)
+
 	if kubeProxyMode == "" && !isSkipProxy {
 		log.V(5).Info("kube proxy mode is not set or skipped, skipping mutation")
 		return nil