feat: enable Cilium kube-proxy replacement for new clusters (#1288)

**What problem does this PR solve?**:
This PR enables Cilium's kube-proxy replacement feature automatically
when clusters disable kube-proxy installation for new clusters.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

---------

Co-authored-by: Jimmi Dyson <[email protected]>

Author: dkoshkin

charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml

@@ -33,3 +33,11 @@ socketLB:
 envoy:
   image:
     useDigest: false
+k8sServiceHost: auto
+{{- with .ControlPlane }}
+{{- range $key, $val := .metadata.annotations }}
+{{- if eq $key "controlplane.cluster.x-k8s.io/skip-kube-proxy" }}
+kubeProxyReplacement: true{{ break }}
+{{- end }}
+{{- end }}
+{{- end }}

examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml

@@ -16,6 +16,9 @@ spec:
   topology:
     class: aws-quick-start
     controlPlane:
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig

examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml

@@ -57,7 +57,9 @@ spec:
   topology:
     class: nutanix-quick-start
     controlPlane:
-      metadata: {}
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig

hack/addons/kustomize/cilium/kustomization.yaml.tmpl

@@ -19,7 +19,8 @@ helmCharts:
   skipTests: true
   namespace: kube-system
   kubeVersion: ${E2E_KUBERNETES_VERSION}
-  valuesFile: ../../../../charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml
+  # This values file will be created by the update-cilium-manifests.sh script when generating the CRS manifests.
+  valuesFile: helm-values.yaml
   # The CRS manifests are generated from the Cilium Helm chart using Kustomize. The Cilium
   # Helm chart uses a Helm hook to generate TLS certificates for Hubble. As the
   # CRS manifests are static those Helm hooks don't apply and so for now Hubble is
@@ -29,5 +30,6 @@ helmCharts:
       enabled: false
       relay:
         enabled: false
+    k8sServiceHost: ""
 
 namespace: kube-system

hack/addons/update-cilium-manifests.sh

@@ -21,13 +21,18 @@ readonly FILE_NAME="cilium.yaml"
 
 readonly KUSTOMIZE_BASE_DIR="${SCRIPT_DIR}/kustomize/cilium"
 mkdir -p "${ASSETS_DIR}/cilium"
-envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${KUSTOMIZE_BASE_DIR}/kustomization.yaml"
-trap_add "rm -f ${KUSTOMIZE_BASE_DIR}/kustomization.yaml" EXIT
+envsubst -no-unset <"${KUSTOMIZE_BASE_DIR}/kustomization.yaml.tmpl" >"${ASSETS_DIR}/kustomization.yaml"
+
+cat <<EOF >"${ASSETS_DIR}/gomplate-context.yaml"
+ControlPlane: {}
+EOF
+gomplate -f "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/addons/cni/cilium/values-template.yaml" \
+  --context .="${ASSETS_DIR}/gomplate-context.yaml" \
+  >"${ASSETS_DIR}/helm-values.yaml"
 
 kustomize build \
   --load-restrictor LoadRestrictionsNone \
-  --enable-helm "${KUSTOMIZE_BASE_DIR}/" >"${ASSETS_DIR}/${FILE_NAME}"
-trap_add "rm -rf ${KUSTOMIZE_BASE_DIR}/charts/" EXIT
+  --enable-helm "${ASSETS_DIR}/" >"${ASSETS_DIR}/${FILE_NAME}"
 
 # The operator manifest in YAML format is pretty big. It turns out that much of that is whitespace. Converting the
 # manifest to JSON without indentation allows us to remove most of the whitespace, reducing the size by more than half.

hack/examples/overlays/clusters/aws/cilium/helm-addon/kustomization.yaml.tmpl

@@ -14,3 +14,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/cilium.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/skip-kube-proxy.yaml

hack/examples/overlays/clusters/nutanix/cilium/helm-addon/kustomization.yaml.tmpl

@@ -14,3 +14,6 @@ patches:
   - target:
       kind: Cluster
     path: ../../../../../patches/cilium.yaml
+  - target:
+      kind: Cluster
+    path: ../../../../../patches/skip-kube-proxy.yaml

hack/examples/patches/skip-kube-proxy.yaml

@@ -0,0 +1,13 @@
+# Copyright 2025 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: not-used
+spec:
+  topology:
+    controlPlane:
+      metadata:
+        annotations:
+          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""

hack/tools/fetch-images/main.go

@@ -259,7 +259,31 @@ func getValuesFileForChartIfNeeded(chartName, carenChartDirectory string) (strin
 	case "snapshot-controller":
 		return filepath.Join(carenChartDirectory, "addons", "csi", "snapshot-controller", defaultHelmAddonFilename), nil
 	case "cilium":
-		return filepath.Join(carenChartDirectory, "addons", "cni", "cilium", defaultHelmAddonFilename), nil
+		f := filepath.Join(carenChartDirectory, "addons", "cni", "cilium", defaultHelmAddonFilename)
+		tempFile, err := os.CreateTemp("", "")
+		if err != nil {
+			return "", fmt.Errorf("failed to create temp file: %w", err)
+		}
+
+		type input struct {
+			ControlPlane map[string]interface{}
+		}
+		templateInput := input{
+			ControlPlane: map[string]interface{}{
+				"metadata": map[string]interface{}{
+					"annotations": map[string]interface{}{
+						"controlplane.cluster.x-k8s.io/skip-kube-proxy": "",
+					},
+				},
+			},
+		}
+
+		err = template.Must(template.New(defaultHelmAddonFilename).ParseFiles(f)).Execute(tempFile, &templateInput)
+		if err != nil {
+			return "", fmt.Errorf("failed to execute helm values template %w", err)
+		}
+
+		return tempFile.Name(), nil
 	// Calico values differ slightly per provider, but that does not have a material imapct on the images required
 	// so we can use the default values file for AWS provider.
 	case "tigera-operator":