feat: auto enable registry addon in workload clusters (#1175)

dkoshkin · web-flow · commit 82bd1353383c · 2025-06-30T07:53:57.000-07:00
**What problem does this PR solve?**:
This adds a new feature gate `AutoEnableWorkloadClusterRegistry=true`
that will automatically enable the registry addon on workload cluster
creation when it is enabled on the management cluster.

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
&lt;!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
--&gt;

**Special notes for your reviewer**:
&lt;!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
--&gt;
diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go
@@ -39,5 +39,6 @@ const (
 	// DNSVariableName is the DNS external patch variable name.
 	DNSVariableName = "dns"
 
-	ClusterUUIDAnnotationKey = APIGroup + "/cluster-uuid"
+	ClusterUUIDAnnotationKey                = APIGroup + "/cluster-uuid"
+	SkipAutoEnablingWorkloadClusterRegistry = APIGroup + "/skip-auto-enabling-workload-cluster-registry"
 )
diff --git a/api/variables/json.go b/api/variables/json.go
@@ -78,3 +78,20 @@ func GetClusterVariableByName(
 	}
 	return nil
 }
+
+// UpdateClusterVariable updates the variable in the list of cluster variables.
+// If the variable does not exist, it appends it to the list.
+func UpdateClusterVariable(
+	variable *clusterv1.ClusterVariable,
+	clusterVariables []clusterv1.ClusterVariable,
+) []clusterv1.ClusterVariable {
+	name := variable.Name
+	for i := range clusterVariables {
+		if clusterVariables[i].Name == name {
+			clusterVariables[i] = *variable
+			return clusterVariables
+		}
+	}
+	clusterVariables = append(clusterVariables, *variable)
+	return clusterVariables
+}
diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/webhooks.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/webhooks.yaml
@@ -8,6 +8,25 @@ metadata:
   annotations:
     cert-manager.io/inject-ca-from: '{{ .Release.Namespace}}/{{ template "chart.name" . }}-admission-tls'
 webhooks:
+  - admissionReviewVersions:
+      - v1
+    clientConfig:
+      service:
+        name: '{{ include "chart.name" . }}-admission'
+        namespace: '{{ .Release.Namespace }}'
+        path: /mutate-v1beta1-addons
+    failurePolicy: Fail
+    name: addons-defaulter.caren.nutanix.com
+    rules:
+      - apiGroups:
+          - cluster.x-k8s.io
+        apiVersions:
+          - '*'
+        operations:
+          - CREATE
+        resources:
+          - clusters
+    sideEffects: None
   - admissionReviewVersions:
       - v1
     clientConfig:
diff --git a/cmd/main.go b/cmd/main.go
@@ -40,6 +40,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/lifecycle"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/nutanix"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/webhook/addons"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/webhook/cluster"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/webhook/preflight"
 	preflightnutanix "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/webhook/preflight/nutanix"
@@ -237,6 +238,10 @@ func main() {
 		Handler: cluster.NewValidator(mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())),
 	})
 
+	mgr.GetWebhookServer().Register("/mutate-v1beta1-addons", &webhook.Admission{
+		Handler: addons.NewDefaulter(mgr.GetClient(), admission.NewDecoder(mgr.GetScheme())),
+	})
+
 	mgr.GetWebhookServer().Register("/preflight-v1beta1-cluster", &webhook.Admission{
 		Handler: preflight.New(mgr.GetClient(), admission.NewDecoder(mgr.GetScheme()),
 			[]preflight.Checker{
diff --git a/docs/content/addons/registry.md b/docs/content/addons/registry.md
@@ -31,6 +31,22 @@ spec:
             registry: {}
 ```
 
+## Registry in the workload cluster
+
+When the registry is enabled in the management cluster, it can also be automatically enabled in the workload cluster.
+To enable this behavior, set the following feature gate on the controller:
+
+```text
+  --feature-gates=AutoEnableWorkloadClusterRegistry=true
+```
+
+It is also possible to disable this behavior by setting the following annotation on the Cluster resource:
+
+```yaml
+annotations:
+   caren.nutanix.com/skip-auto-enabling-workload-cluster-registry: "true"
+```
+
 ## Registry Certificate
 
 1. A root CA Certificate is deployed in the provider's namespace.
diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go
@@ -7,9 +7,13 @@ import (
 	"k8s.io/component-base/featuregate"
 )
 
+const AutoEnableWorkloadClusterRegistry featuregate.Feature = "AutoEnableWorkloadClusterRegistry"
+
 // defaultFeatureGates returns all known feature gates.
 // To add a new feature, define a key for it above and add it here. The features will be
 // available throughout the codebase.
 func defaultFeatureGates() map[featuregate.Feature]featuregate.FeatureSpec {
-	return map[featuregate.Feature]featuregate.FeatureSpec{}
+	return map[featuregate.Feature]featuregate.FeatureSpec{
+		AutoEnableWorkloadClusterRegistry: {Default: false, PreRelease: featuregate.Alpha},
+	}
 }
diff --git a/pkg/feature/gates.go b/pkg/feature/gates.go
@@ -20,8 +20,8 @@ var (
 	//   featuregatetesting "k8s.io/component-base/featuregate/testing"
 	//   featuregatetesting.SetFeatureGateDuringTest(
 	//     t,
-	//     features.Gates,
-	//     features.<FeatureName>,
+	//     feature.Gates,
+	//     feature.<FeatureName>,
 	//     <value>,
 	//   )()
 	MutableGates featuregate.MutableFeatureGate = featuregate.NewFeatureGate()
diff --git a/pkg/webhook/addons/defaulter.go b/pkg/webhook/addons/defaulter.go
@@ -0,0 +1,30 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package addons
+
+import (
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/feature"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/webhook/addons/registry"
+)
+
+func NewDefaulter(client ctrlclient.Client, decoder admission.Decoder) admission.Handler {
+	return admission.MultiMutatingHandler(
+		allHandlers(client, decoder)...,
+	)
+}
+
+// allHandlers returns a list of all defaulter handlers that should be registered,
+// including any feature-gated handlers.
+func allHandlers(
+	client ctrlclient.Client, decoder admission.Decoder,
+) []admission.Handler {
+	var handlers []admission.Handler
+	if feature.Gates.Enabled(feature.AutoEnableWorkloadClusterRegistry) {
+		handlers = append(handlers, registry.NewWorkloadClusterAutoEnabler(client, decoder).Defaulter())
+	}
+	return handlers
+}
diff --git a/pkg/webhook/addons/defaulter_test.go b/pkg/webhook/addons/defaulter_test.go
@@ -0,0 +1,29 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package addons
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	featuregatetesting "k8s.io/component-base/featuregate/testing"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/feature"
+)
+
+// Test_allHandlers is a crude test to ensure handlers are only registered when the feature gate is enabled.
+func Test_allHandlers(t *testing.T) {
+	handlers := allHandlers(nil, nil)
+	assert.Empty(t, handlers)
+
+	// Enable the feature gates and test again.
+	featuregatetesting.SetFeatureGateDuringTest(
+		t,
+		feature.Gates,
+		feature.AutoEnableWorkloadClusterRegistry,
+		true,
+	)
+	handlers = allHandlers(nil, nil)
+	assert.Len(t, handlers, 1)
+}
diff --git a/pkg/webhook/addons/doc.go b/pkg/webhook/addons/doc.go
@@ -0,0 +1,5 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// +kubebuilder:webhook:path=/mutate-v1beta1-addons,mutating=true,failurePolicy=fail,groups="cluster.x-k8s.io",resources=clusters,verbs=create,versions=*,name=addons-defaulter.caren.nutanix.com,admissionReviewVersions=v1,sideEffects=None
+package addons
diff --git a/pkg/webhook/addons/registry/autoenabler.go b/pkg/webhook/addons/registry/autoenabler.go
@@ -0,0 +1,185 @@
+// Copyright 2025 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package registry
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+
+	carenv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/variables"
+	capiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils"
+)
+
+type workloadClusterAutoEnabler struct {
+	client  ctrlclient.Client
+	decoder admission.Decoder
+}
+
+func NewWorkloadClusterAutoEnabler(
+	client ctrlclient.Client, decoder admission.Decoder,
+) *workloadClusterAutoEnabler {
+	return &workloadClusterAutoEnabler{
+		client:  client,
+		decoder: decoder,
+	}
+}
+
+func (a *workloadClusterAutoEnabler) Defaulter() admission.HandlerFunc {
+	return a.defaulter
+}
+
+func (a *workloadClusterAutoEnabler) defaulter(
+	ctx context.Context,
+	req admission.Request,
+) admission.Response {
+	cluster := &clusterv1.Cluster{}
+	err := a.decoder.Decode(req, cluster)
+	if err != nil {
+		return admission.Errored(http.StatusBadRequest, err)
+	}
+
+	if hasSkipAnnotation(cluster) {
+		return admission.Allowed("")
+	}
+
+	if cluster.Spec.Topology == nil {
+		return admission.Allowed("")
+	}
+
+	// Check if the addon is already enabled in the cluster, if it is just return.
+	clusterRegistry, err := registryAddonFromCluster(cluster)
+	if err != nil {
+		return admission.Errored(
+			http.StatusInternalServerError,
+			fmt.Errorf(
+				"failed to check if registry addon is enabled in cluster: %w",
+				err,
+			))
+	}
+	if clusterRegistry != nil {
+		return admission.Allowed("")
+	}
+
+	// Check if the global image registry mirror is enabled in the cluster, if it is just return.
+	globalImageRegistryMirror, err := globalImageRegistryMirrorFromCluster(cluster)
+	if err != nil {
+		return admission.Errored(
+			http.StatusInternalServerError,
+			fmt.Errorf(
+				"failed to check if global image registry mirror is enabled in cluster: %w",
+				err,
+			))
+	}
+	if globalImageRegistryMirror != nil {
+		return admission.Allowed("")
+	}
+
+	managementCluster, err := capiutils.ManagementCluster(ctx, a.client)
+	if err != nil {
+		return admission.Errored(
+			http.StatusInternalServerError,
+			fmt.Errorf(
+				"failed to get management cluster: %w",
+				err,
+			))
+	}
+	// Check if creating a workload cluster, ie managementCluster is not nil. If it is nil just return.
+	if managementCluster == nil {
+		return admission.Allowed("")
+	}
+	// Check if the addon is enabled in the management cluster, if not just return.
+	managementClusterRegistry, err := registryAddonFromCluster(managementCluster)
+	if err != nil {
+		return admission.Errored(
+			http.StatusInternalServerError,
+			fmt.Errorf(
+				"failed to check if registry addon is enabled in management cluster: %w",
+				err,
+			))
+	}
+	if managementClusterRegistry == nil {
+		return admission.Allowed("")
+	}
+
+	// If the registry addon is not enabled in the cluster and is enabled in the management cluster, enable it here.
+	err = enabledSameRegistryAddonInCluster(cluster, managementClusterRegistry)
+	if err != nil {
+		return admission.Errored(
+			http.StatusInternalServerError,
+			fmt.Errorf(
+				"failed to enable registry addon in cluster: %w",
+				err,
+			))
+	}
+
+	marshaledCluster, err := json.Marshal(cluster)
+	if err != nil {
+		return admission.Errored(http.StatusInternalServerError, err)
+	}
+	return admission.PatchResponseFromRaw(req.Object.Raw, marshaledCluster)
+}
+
+func hasSkipAnnotation(cluster *clusterv1.Cluster) bool {
+	if cluster.Annotations == nil {
+		return false
+	}
+	_, ok := cluster.Annotations[carenv1.SkipAutoEnablingWorkloadClusterRegistry]
+	return ok
+}
+
+func registryAddonFromCluster(cluster *clusterv1.Cluster) (*carenv1.RegistryAddon, error) {
+	spec, err := variables.UnmarshalClusterConfigVariable(cluster.Spec.Topology.Variables)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal cluster variable: %w", err)
+	}
+	if spec == nil {
+		return nil, nil
+	}
+	if spec.Addons == nil {
+		return nil, nil
+	}
+
+	return spec.Addons.Registry, nil
+}
+
+func globalImageRegistryMirrorFromCluster(cluster *clusterv1.Cluster) (*carenv1.GlobalImageRegistryMirror, error) {
+	spec, err := variables.UnmarshalClusterConfigVariable(cluster.Spec.Topology.Variables)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal cluster variable: %w", err)
+	}
+	if spec == nil {
+		return nil, nil
+	}
+
+	return spec.GlobalImageRegistryMirror, nil
+}
+
+func enabledSameRegistryAddonInCluster(cluster *clusterv1.Cluster, sourceAddon *carenv1.RegistryAddon) error {
+	spec, err := variables.UnmarshalClusterConfigVariable(cluster.Spec.Topology.Variables)
+	if err != nil {
+		return fmt.Errorf("failed to unmarshal cluster variable: %w", err)
+	}
+
+	if spec.Addons == nil {
+		spec.Addons = &variables.Addons{}
+	}
+	spec.Addons.Registry = &carenv1.RegistryAddon{
+		Provider: sourceAddon.Provider,
+	}
+
+	variable, err := variables.MarshalToClusterVariable(carenv1.ClusterConfigVariableName, spec)
+	if err != nil {
+		return fmt.Errorf("failed to marshal cluster variable: %w", err)
+	}
+	cluster.Spec.Topology.Variables = variables.UpdateClusterVariable(variable, cluster.Spec.Topology.Variables)
+
+	return nil
+}
diff --git a/pkg/webhook/addons/registry/webhook_suite_test.go b/pkg/webhook/addons/registry/webhook_suite_test.go
diff --git a/pkg/webhook/addons/registry/webhook_test.go b/pkg/webhook/addons/registry/webhook_test.go

Original file line number	Diff line number	Diff line change
`@@ -39,5 +39,6 @@ const (`
`39`	`39`	`// DNSVariableName is the DNS external patch variable name.`
`40`	`40`	`DNSVariableName = "dns"`
`41`	`41`
`42`		`- ClusterUUIDAnnotationKey = APIGroup + "/cluster-uuid"`
	`42`	`+ ClusterUUIDAnnotationKey = APIGroup + "/cluster-uuid"`
	`43`	`+ SkipAutoEnablingWorkloadClusterRegistry = APIGroup + "/skip-auto-enabling-workload-cluster-registry"`
`43`	`44`	`)`
Original file line number	Diff line number	Diff line change
`@@ -7,9 +7,13 @@ import (`
`7`	`7`	`"k8s.io/component-base/featuregate"`
`8`	`8`	`)`
`9`	`9`
	`10`	`+const AutoEnableWorkloadClusterRegistry featuregate.Feature = "AutoEnableWorkloadClusterRegistry"`
	`11`	`+`
`10`	`12`	`// defaultFeatureGates returns all known feature gates.`
`11`	`13`	`// To add a new feature, define a key for it above and add it here. The features will be`
`12`	`14`	`// available throughout the codebase.`
`13`	`15`	`func defaultFeatureGates() map[featuregate.Feature]featuregate.FeatureSpec {`
`14`		`- return map[featuregate.Feature]featuregate.FeatureSpec{}`
	`16`	`+ return map[featuregate.Feature]featuregate.FeatureSpec{`
	`17`	`+ AutoEnableWorkloadClusterRegistry: {Default: false, PreRelease: featuregate.Alpha},`
	`18`	`+ }`
`15`	`19`	`}`