feat: Allow skipping kube-proxy on cluster creation

This commit allows users to skip kube-proxy installation when
creating clusters. This is enforced via CEL to prevent users
from moving between kube-proxy and non-kube-proxy deployments.

Follow up work will allow Cilium configuration to enable their
kube-proxy replacement (already possible via custom Helm values)
and migration from kube-proxy to kube-proxy replacement for
existing clusters.

Author: jimmidyson

api/v1alpha1/clusterconfig_types.go

@@ -212,6 +212,14 @@ type GenericClusterConfigSpec struct {
 
 	// +kubebuilder:validation:Optional
 	DNS *DNS `json:"dns,omitempty"`
+
+	// SkipKubeProxy indicates whether to skip the installation of kube-proxy.
+	// This is useful in scenarios where kube-proxy is not required or managed
+	// externally.
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default=false
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value cannot be changed after cluster creation"
+	SkipKubeProxy bool `json:"skipKubeProxy,omitempty"`
 }
 
 type Image struct {

api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

@@ -583,6 +583,16 @@ spec:
                       description: HTTPS proxy value.
                       type: string
                   type: object
+                skipKubeProxy:
+                  default: false
+                  description: |-
+                    SkipKubeProxy indicates whether to skip the installation of kube-proxy.
+                    This is useful in scenarios where kube-proxy is not required or managed
+                    externally.
+                  type: boolean
+                  x-kubernetes-validations:
+                    - message: Value cannot be changed after cluster creation
+                      rule: self == oldSelf
                 users:
                   items:
                     description: User defines the input for a generated user in cloud-init.

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

@@ -520,6 +520,16 @@ spec:
                       description: HTTPS proxy value.
                       type: string
                   type: object
+                skipKubeProxy:
+                  default: false
+                  description: |-
+                    SkipKubeProxy indicates whether to skip the installation of kube-proxy.
+                    This is useful in scenarios where kube-proxy is not required or managed
+                    externally.
+                  type: boolean
+                  x-kubernetes-validations:
+                    - message: Value cannot be changed after cluster creation
+                      rule: self == oldSelf
                 users:
                   items:
                     description: User defines the input for a generated user in cloud-init.

api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml

@@ -198,6 +198,16 @@ spec:
                     description: HTTPS proxy value.
                     type: string
                 type: object
+              skipKubeProxy:
+                default: false
+                description: |-
+                  SkipKubeProxy indicates whether to skip the installation of kube-proxy.
+                  This is useful in scenarios where kube-proxy is not required or managed
+                  externally.
+                type: boolean
+                x-kubernetes-validations:
+                - message: Value cannot be changed after cluster creation
+                  rule: self == oldSelf
               users:
                 items:
                   description: User defines the input for a generated user in cloud-init.

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -793,6 +793,16 @@ spec:
                       description: HTTPS proxy value.
                       type: string
                   type: object
+                skipKubeProxy:
+                  default: false
+                  description: |-
+                    SkipKubeProxy indicates whether to skip the installation of kube-proxy.
+                    This is useful in scenarios where kube-proxy is not required or managed
+                    externally.
+                  type: boolean
+                  x-kubernetes-validations:
+                    - message: Value cannot be changed after cluster creation
+                      rule: self == oldSelf
                 users:
                   items:
                     description: User defines the input for a generated user in cloud-init.

docs/content/customization/generic/skip-kube-proxy.md

@@ -0,0 +1,37 @@
++++
+title = "Skip deployment of kube-proxy"
++++
+
+When deploying a CNI implementation that can replace `kube-proxy`, it is necessary to disable `kube-proxy` to avoid
+potential conflicts. By default, `kube-proxy` is enabled.
+
+## Example
+
+To disable the deployment of `kube-proxy`, specify the following configuration:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          skipKubeProxy: true
+```
+
+Applying this configuration will result in the following configuration being applied:
+
+- `KubeadmControlPlaneTemplate`:
+
+  - ```yaml
+    spec:
+      template:
+        spec:
+          kubeadmConfigSpec:
+            initConfiguration:
+              skipPhases:
+                - addon/kube-proxy
+    ```

pkg/handlers/generic/mutation/handlers.go

@@ -23,6 +23,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/noderegistration"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/skipkubeproxy"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/taints"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/users"
 )
@@ -44,6 +45,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
 		containerdunprivilegedports.NewPatch(),
 		encryptionatrest.NewPatch(mgr.GetClient(), encryptionatrest.RandomTokenGenerator),
 		autorenewcerts.NewPatch(),
+		skipkubeproxy.NewPatch(),
 
 		// Some patches may have changed containerd configuration.
 		// We write the configuration changes to disk, and must run a command

pkg/handlers/generic/mutation/skipkubeproxy/inject.go

@@ -0,0 +1,113 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package skipkubeproxy
+
+import (
+	"context"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
+)
+
+const (
+	// VariableName is the external patch variable name.
+	VariableName = "skipKubeProxy"
+)
+
+type skipKubeProxy struct {
+	variableName      string
+	variableFieldPath []string
+}
+
+func NewPatch() *skipKubeProxy {
+	return newSkipKubeProxy(
+		v1alpha1.ClusterConfigVariableName,
+		VariableName,
+	)
+}
+
+func newSkipKubeProxy(
+	variableName string,
+	variableFieldPath ...string,
+) *skipKubeProxy {
+	return &skipKubeProxy{
+		variableName:      variableName,
+		variableFieldPath: variableFieldPath,
+	}
+}
+
+func (h *skipKubeProxy) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	_ client.ObjectKey,
+	clusterGetter mutation.ClusterGetter,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	skipKubeProxy, err := variables.Get[bool](
+		vars,
+		h.variableName,
+		h.variableFieldPath...,
+	)
+	if err != nil {
+		if variables.IsNotFoundError(err) {
+			log.V(5).Info("Control Plane skip kube proxy variable not defined")
+			return nil
+		}
+
+		return err
+	}
+
+	log = log.WithValues(
+		"variableName",
+		h.variableName,
+		"variableFieldPath",
+		h.variableFieldPath,
+		"variableValue",
+		skipKubeProxy,
+	)
+
+	if !skipKubeProxy {
+		log.V(5).Info("Skip kube proxy variable is false, skipping mutation")
+		return nil
+	}
+
+	return patches.MutateIfApplicable(
+		obj,
+		vars,
+		&holderRef,
+		selectors.ControlPlane(),
+		log,
+		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", client.ObjectKeyFromObject(obj),
+			).Info("adding skip kube proxy addon to control plane kubeadm config spec")
+			if obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration == nil {
+				obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration = &bootstrapv1.InitConfiguration{}
+			}
+			obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.SkipPhases = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.InitConfiguration.SkipPhases,
+				"addon/kube-proxy",
+			)
+
+			return nil
+		},
+	)
+}

pkg/handlers/generic/mutation/skipkubeproxy/inject_test.go

@@ -0,0 +1,111 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package skipkubeproxy
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
+)
+
+func TestAutoRenewCertsPatch(t *testing.T) {
+	gomega.RegisterFailHandler(Fail)
+	RunSpecs(t, "Auto renew certs mutator suite")
+}
+
+type testObj struct {
+	patchTest capitest.PatchTestDef
+}
+
+var _ = Describe("Generate skip kube proxy patches", func() {
+	patchGenerator := func() mutation.GeneratePatches {
+		return mutation.NewMetaGeneratePatchesHandler("", nil, NewPatch()).(mutation.GeneratePatches)
+	}
+
+	testDefs := []testObj{
+		{
+			patchTest: capitest.PatchTestDef{
+				Name: "skip kube proxy set with AWS",
+				Vars: []runtimehooksv1.Variable{
+					capitest.VariableWithValue(
+						v1alpha1.ClusterConfigVariableName,
+						v1alpha1.AWSClusterConfigSpec{
+							GenericClusterConfigSpec: v1alpha1.GenericClusterConfigSpec{
+								SkipKubeProxy: true,
+							},
+						},
+					),
+				},
+				RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+				ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/initConfiguration/skipPhases",
+					ValueMatcher: gomega.ConsistOf([]string{
+						"addon/kube-proxy",
+					}),
+				}},
+			},
+		},
+		{
+			patchTest: capitest.PatchTestDef{
+				Name: "skip kube proxy set with Docker",
+				Vars: []runtimehooksv1.Variable{
+					capitest.VariableWithValue(
+						v1alpha1.ClusterConfigVariableName,
+						v1alpha1.DockerClusterConfigSpec{
+							GenericClusterConfigSpec: v1alpha1.GenericClusterConfigSpec{
+								SkipKubeProxy: true,
+							},
+						},
+					),
+				},
+				RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+				ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/initConfiguration/skipPhases",
+					ValueMatcher: gomega.ConsistOf([]string{
+						"addon/kube-proxy",
+					}),
+				}},
+			},
+		},
+		{
+			patchTest: capitest.PatchTestDef{
+				Name: "skip kube proxy set with Nutanix",
+				Vars: []runtimehooksv1.Variable{
+					capitest.VariableWithValue(
+						v1alpha1.ClusterConfigVariableName,
+						v1alpha1.NutanixClusterConfigSpec{
+							GenericClusterConfigSpec: v1alpha1.GenericClusterConfigSpec{
+								SkipKubeProxy: true,
+							},
+						},
+					),
+				},
+				RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+				ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/initConfiguration/skipPhases",
+					ValueMatcher: gomega.ConsistOf([]string{
+						"addon/kube-proxy",
+					}),
+				}},
+			},
+		},
+	}
+
+	// create test node for each case
+	for _, tt := range testDefs {
+		It(tt.patchTest.Name, func() {
+			capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt.patchTest)
+		})
+	}
+})