refactor: new disabled kube-proxy mode

dkoshkin · dkoshkin · commit 0ac15301ddd3 · 2025-09-25T08:09:46.000-07:00
diff --git a/api/v1alpha1/clusterconfig_types.go b/api/v1alpha1/clusterconfig_types.go
@@ -437,14 +437,17 @@ const (
 	// KubeProxyModeNFTables indicates that kube-proxy should be installed in nftables
 	// mode.
 	KubeProxyModeNFTables KubeProxyMode = "nftables"
+	// KubeProxyModeDisabled indicates that kube-proxy should be disabled.
+	KubeProxyModeDisabled KubeProxyMode = "disabled"
 )
 
 type KubeProxy struct {
 	// Mode specifies the mode for kube-proxy:
 	// - iptables means that kube-proxy is installed in iptables mode.
 	// - nftables means that kube-proxy is installed in nftables mode.
+	// - disabled means that kube-proxy is disabled.
 	// +kubebuilder:validation:Optional
-	// +kubebuilder:validation:Enum=iptables;nftables
+	// +kubebuilder:validation:Enum=iptables;nftables;disabled
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value cannot be changed after cluster creation"
 	Mode KubeProxyMode `json:"mode,omitempty"`
 }
diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml
@@ -716,9 +716,11 @@ spec:
                         Mode specifies the mode for kube-proxy:
                         - iptables means that kube-proxy is installed in iptables mode.
                         - nftables means that kube-proxy is installed in nftables mode.
+                        - disabled means that kube-proxy is disabled.
                       enum:
                         - iptables
                         - nftables
+                        - disabled
                       type: string
                       x-kubernetes-validations:
                         - message: Value cannot be changed after cluster creation
diff --git a/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml
@@ -531,9 +531,11 @@ spec:
                         Mode specifies the mode for kube-proxy:
                         - iptables means that kube-proxy is installed in iptables mode.
                         - nftables means that kube-proxy is installed in nftables mode.
+                        - disabled means that kube-proxy is disabled.
                       enum:
                         - iptables
                         - nftables
+                        - disabled
                       type: string
                       x-kubernetes-validations:
                         - message: Value cannot be changed after cluster creation
diff --git a/api/v1alpha1/crds/caren.nutanix.com_eksclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_eksclusterconfigs.yaml
@@ -443,9 +443,11 @@ spec:
                       Mode specifies the mode for kube-proxy:
                       - iptables means that kube-proxy is installed in iptables mode.
                       - nftables means that kube-proxy is installed in nftables mode.
+                      - disabled means that kube-proxy is disabled.
                     enum:
                     - iptables
                     - nftables
+                    - disabled
                     type: string
                     x-kubernetes-validations:
                     - message: Value cannot be changed after cluster creation
diff --git a/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml
@@ -719,9 +719,11 @@ spec:
                         Mode specifies the mode for kube-proxy:
                         - iptables means that kube-proxy is installed in iptables mode.
                         - nftables means that kube-proxy is installed in nftables mode.
+                        - disabled means that kube-proxy is disabled.
                       enum:
                         - iptables
                         - nftables
+                        - disabled
                       type: string
                       x-kubernetes-validations:
                         - message: Value cannot be changed after cluster creation
diff --git a/api/variables/getters.go b/api/variables/getters.go
@@ -27,3 +27,29 @@ func RegistryAddon(cluster *clusterv1.Cluster) (*carenv1.RegistryAddon, error) {
 
 	return spec.Addons.Registry, nil
 }
+
+// KubeProxyMode retrieves the kube-proxy mode from the cluster's topology variables.
+// Returns nil if the kube-proxy mode is not defined.
+func KubeProxyMode(cluster *clusterv1.Cluster) (*carenv1.KubeProxyMode, error) {
+	spec, err := UnmarshalClusterConfigVariable(cluster.Spec.Topology.Variables)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal cluster variable: %w", err)
+	}
+	if spec == nil {
+		return nil, nil
+	}
+	if spec.KubeProxy == nil {
+		return nil, nil
+	}
+
+	return &spec.KubeProxy.Mode, nil
+}
+
+// KubeProxyIsDisabled returns true if kube-proxy mode from the cluster's topology variables is disabled.
+func KubeProxyIsDisabled(cluster *clusterv1.Cluster) (bool, error) {
+	mode, err := KubeProxyMode(cluster)
+	if err != nil {
+		return false, err
+	}
+	return mode != nil && *mode == carenv1.KubeProxyModeDisabled, nil
+}
diff --git a/common/pkg/capi/utils/annotations.go b/common/pkg/capi/utils/annotations.go
diff --git a/docs/content/customization/generic/kube-proxy-mode.md b/docs/content/customization/generic/kube-proxy-mode.md
@@ -60,10 +60,11 @@ metadata:
   name: <NAME>
 spec:
   topology:
-    controlPlane:
-      metadata:
-        annotations:
-          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
+    variables:
+      - name: clusterConfig
+        value:
+          kubeProxy:
+            mode: disabled
 ```
 
 Applying this configuration will result in the following configuration being applied:
diff --git a/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml
@@ -16,9 +16,6 @@ spec:
   topology:
     class: aws-quick-start
     controlPlane:
-      metadata:
-        annotations:
-          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig
@@ -52,6 +49,8 @@ spec:
         encryptionAtRest:
           providers:
           - aescbc: {}
+        kubeProxy:
+          mode: disabled
     - name: workerConfig
       value:
         aws:
diff --git a/examples/capi-quick-start/eks-cluster.yaml b/examples/capi-quick-start/eks-cluster.yaml
@@ -62,10 +62,7 @@ metadata:
 spec:
   topology:
     class: eks-quick-start
-    controlPlane:
-      metadata:
-        annotations:
-          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
+    controlPlane: {}
     variables:
     - name: clusterConfig
       value:
@@ -89,6 +86,8 @@ spec:
           nfd: {}
         eks:
           region: us-west-2
+        kubeProxy:
+          mode: disabled
     version: ${KUBERNETES_VERSION}
     workers:
       machineDeployments:
diff --git a/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml
@@ -57,9 +57,7 @@ spec:
   topology:
     class: nutanix-quick-start
     controlPlane:
-      metadata:
-        annotations:
-          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
+      metadata: {}
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig
@@ -124,6 +122,8 @@ spec:
             secretRef:
               name: ${CLUSTER_NAME}-dockerhub-credentials
           url: https://docker.io
+        kubeProxy:
+          mode: disabled
         nutanix:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
diff --git a/examples/capi-quick-start/nutanix-cluster-with-failuredomains-cilium-helm-addon.yaml b/examples/capi-quick-start/nutanix-cluster-with-failuredomains-cilium-helm-addon.yaml
@@ -93,9 +93,7 @@ spec:
   topology:
     class: nutanix-quick-start
     controlPlane:
-      metadata:
-        annotations:
-          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
+      metadata: {}
       replicas: ${CONTROL_PLANE_MACHINE_COUNT}
     variables:
     - name: clusterConfig
@@ -158,6 +156,8 @@ spec:
             secretRef:
               name: ${CLUSTER_NAME}-dockerhub-credentials
           url: https://docker.io
+        kubeProxy:
+          mode: disabled
         nutanix:
           controlPlaneEndpoint:
             host: ${CONTROL_PLANE_ENDPOINT_IP}
diff --git a/hack/examples/patches/skip-kube-proxy.yaml b/hack/examples/patches/skip-kube-proxy.yaml
@@ -1,13 +1,7 @@
 # Copyright 2025 Nutanix. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-apiVersion: cluster.x-k8s.io/v1beta1
-kind: Cluster
-metadata:
-  name: not-used
-spec:
-  topology:
-    controlPlane:
-      metadata:
-        annotations:
-          controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
+- op: "add"
+  path: "/spec/topology/variables/0/value/kubeProxy"
+  value:
+    mode: disabled
diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/handler.go b/pkg/handlers/generic/lifecycle/cni/cilium/handler.go
@@ -21,10 +21,10 @@ import (
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables"
 	commonhandlers "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
-	capiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/addons"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/config"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
@@ -264,8 +264,13 @@ func runApply(
 	// It is possible to disable kube-proxy and migrate to Cilium's kube-proxy replacement feature in a running cluster.
 	// In this case, we need to wait for Cilium to be restarted with new configuration and then cleanup kube-proxy.
 
-	// If skip kube-proxy is not set, return early.
-	if !capiutils.ShouldSkipKubeProxy(cluster) {
+	// If kube-proxy is not disabled, return early.
+	kubeProxyIsDisabled, err := apivariables.KubeProxyIsDisabled(cluster)
+	if err != nil {
+		return fmt.Errorf("failed to get kube proxy mode: %w", err)
+	}
+
+	if !kubeProxyIsDisabled {
 		return nil
 	}
 
diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/handler_integration_test.go b/pkg/handlers/generic/lifecycle/cni/cilium/handler_integration_test.go
@@ -19,9 +19,10 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/controllers/remote"
-	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle/addons"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
 )
@@ -38,7 +39,7 @@ var _ = Describe("Test runApply", func() {
 		cluster, remoteClient := setupTestCluster(ctx, c)
 		strategy := addons.NewTestStrategy(nil)
 
-		By("Should not delete kube-proxy when skip kube-proxy is not set")
+		By("Should not delete kube-proxy is not disabled")
 		err = runApply(ctx, c, cluster, strategy, cluster.Namespace, logr.Discard())
 		Expect(err).To(BeNil())
 
@@ -71,10 +72,10 @@ var _ = Describe("Test runApply", func() {
 		Expect(err).To(BeNil())
 		Expect(configMap).ToNot(BeNil())
 
-		By("Should not delete kube-proxy when Cilium DaemonSet is not updated")
-		cluster.Spec.Topology.ControlPlane.Metadata.Annotations = map[string]string{
-			controlplanev1.SkipKubeProxyAnnotation: "",
-		}
+		By("Should delete kube-proxy when kube-proxy is disabled")
+		err = disableKubeProxy(cluster)
+		Expect(err).To(BeNil())
+
 		// Speed up the test.
 		waitTimeout = 1 * time.Second
 		err = runApply(ctx, c, cluster, strategy, cluster.Namespace, logr.Discard())
@@ -264,3 +265,27 @@ func setupTestCluster(
 
 	return cluster, remoteClient
 }
+
+func disableKubeProxy(cluster *clusterv1.Cluster) error {
+	spec, err := apivariables.UnmarshalClusterConfigVariable(cluster.Spec.Topology.Variables)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal cluster variable: %w", err)
+	}
+
+	if spec == nil {
+		spec = &apivariables.ClusterConfigSpec{}
+	}
+	if spec.KubeProxy == nil {
+		spec.KubeProxy = &v1alpha1.KubeProxy{
+			Mode: v1alpha1.KubeProxyModeDisabled,
+		}
+	}
+
+	variable, err := apivariables.MarshalToClusterVariable(v1alpha1.ClusterConfigVariableName, spec)
+	if err != nil {
+		return fmt.Errorf("failed to marshal cluster variable: %w", err)
+	}
+	cluster.Spec.Topology.Variables = apivariables.UpdateClusterVariable(variable, cluster.Spec.Topology.Variables)
+
+	return nil
+}
diff --git a/pkg/handlers/generic/lifecycle/cni/cilium/template.go b/pkg/handlers/generic/lifecycle/cni/cilium/template.go
@@ -10,11 +10,16 @@ import (
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
-	capiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils"
+	apivariables "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables"
 )
 
 // templateValues enables kube-proxy replacement when kube-proxy is disabled.
 func templateValues(cluster *clusterv1.Cluster, text string) (string, error) {
+	kubeProxyIsDisabled, err := apivariables.KubeProxyIsDisabled(cluster)
+	if err != nil {
+		return "", fmt.Errorf("failed to check if kube-proxy is disabled: %w", err)
+	}
+
 	ciliumTemplate, err := template.New("").Parse(text)
 	if err != nil {
 		return "", fmt.Errorf("failed to parse template: %w", err)
@@ -24,9 +29,9 @@ func templateValues(cluster *clusterv1.Cluster, text string) (string, error) {
 		EnableKubeProxyReplacement bool
 	}
 
-	// Assume when kube-proxy is skipped, we should enable Cilium's kube-proxy replacement feature.
+	// Assume when kube-proxy is disabled, we should enable Cilium's kube-proxy replacement feature.
 	templateInput := input{
-		EnableKubeProxyReplacement: capiutils.ShouldSkipKubeProxy(cluster),
+		EnableKubeProxyReplacement: kubeProxyIsDisabled,
 	}
 
 	var b bytes.Buffer
diff --git a/pkg/handlers/generic/mutation/kubeproxymode/inject.go b/pkg/handlers/generic/mutation/kubeproxymode/inject.go
diff --git a/pkg/handlers/generic/mutation/kubeproxymode/inject_test.go b/pkg/handlers/generic/mutation/kubeproxymode/inject_test.go
