feat(stacked-PR): create cluster with custom helm chart config

api/v1alpha1/addon_types.go

@@ -86,6 +86,9 @@ type NutanixAddons struct {
 }
 
 type GenericAddons struct {
+	// +kubebuilder:validation:Optional
+	HelmChartConfig *HelmChartConfig `json:"helmChartConfig,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	CNI *CNI `json:"cni,omitempty"`
 
@@ -102,6 +105,12 @@ type GenericAddons struct {
 	ServiceLoadBalancer *ServiceLoadBalancer `json:"serviceLoadBalancer,omitempty"`
 }
 
+type HelmChartConfig struct {
+	// Reference to a ConfigMap containing configuration for addons Helm charts.
+	// +kubebuilder:validation:Required
+	ConfigMapRef LocalObjectReference `json:"configMapRef"`
+}
+
 type AddonStrategy string
 
 // CNI required for providing CNI configuration.

api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

@@ -201,6 +201,23 @@ spec:
                         - defaultStorage
                         - providers
                       type: object
+                    helmChartConfig:
+                      properties:
+                        configMapRef:
+                          description: Reference to a ConfigMap containing configuration for addons Helm charts.
+                          properties:
+                            name:
+                              description: |-
+                                Name of the referent.
+                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              minLength: 1
+                              type: string
+                          required:
+                            - name
+                          type: object
+                      required:
+                        - configMapRef
+                      type: object
                     nfd:
                       description: NFD tells us to enable or disable the node feature discovery addon.
                       properties:

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

@@ -210,6 +210,23 @@ spec:
                         - defaultStorage
                         - providers
                       type: object
+                    helmChartConfig:
+                      properties:
+                        configMapRef:
+                          description: Reference to a ConfigMap containing configuration for addons Helm charts.
+                          properties:
+                            name:
+                              description: |-
+                                Name of the referent.
+                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              minLength: 1
+                              type: string
+                          required:
+                            - name
+                          type: object
+                      required:
+                        - configMapRef
+                      type: object
                     nfd:
                       description: NFD tells us to enable or disable the node feature discovery addon.
                       properties:

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -210,6 +210,23 @@ spec:
                         - defaultStorage
                         - providers
                       type: object
+                    helmChartConfig:
+                      properties:
+                        configMapRef:
+                          description: Reference to a ConfigMap containing configuration for addons Helm charts.
+                          properties:
+                            name:
+                              description: |-
+                                Name of the referent.
+                                More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                              minLength: 1
+                              type: string
+                          required:
+                            - name
+                          type: object
+                      required:
+                        - configMapRef
+                      type: object
                     nfd:
                       description: NFD tells us to enable or disable the node feature discovery addon.
                       properties:

docs/content/addons/custom-helm-config.md

@@ -0,0 +1,72 @@
++++
+title = "Custom Helm Chart Configuration"
+icon = "fa-solid fa-eye"
++++
+
+A default helm chart configuration is provided with a ConfigMap `default-helm-addons-config`.
+This ConfigMap contains helm chart URL and version for each addon.
+A a ConfigMap with customized addon configuration can be created and referenced in the `Cluster` object.
+
+The helm chart configuration for an addon included in the custom ConfigMap will be installed on the cluster.
+If configuration for an addon is not included in the the customized addon configuration ConfigMap,
+it will be defauled from the `default-helm-addons-config` configmap.
+
+The content of the ConfigMap must use below specification:
+
+- It should be immutable
+- The format of the addon chart configuration should be:
+
+```yaml
+  addon-name: |
+    ChartName: addon-chart-name
+    ChartVersion: 1.0.0
+    RepositoryURL: https://my-chart-repository.example.com/charts/
+```
+
+Example configmap `custom-helm-addons-config.yaml`:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: <CLUSTER_NAME>-helm-addons-config-<uuid>
+  namespace: <CLUSTER_NAMESPACE>
+  labels:
+    clusterctl.cluster.x-k8s.io/move: ""
+immutable: true
+data:
+  nutanix-ccm: |
+    ChartName: nutanix-cloud-provider
+    ChartVersion: 0.4.2
+    RepositoryURL: https://nutanix.github.io/helm/
+  nutanix-storage-csi: |
+    ChartName: nutanix-csi-storage
+    ChartVersion: 3.2.0
+    RepositoryURL: https://nutanix.github.io/helm-releases/
+```
+
+create the custom configmap in the same namespace as the `Cluster`
+
+```shell
+kubectl create -f custom-helm-addons-config.yaml
+```
+
+## Example
+
+To install addons using custom helm configuration, specify following values:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          addons:
+            helmChartConfig:
+              configMapRef:
+                name: <NAME>-helm-addons-config
+```

pkg/handlers/generic/lifecycle/ccm/aws/handler.go

@@ -95,7 +95,7 @@ func (a *AWSCCM) Apply(
 	var strategy addons.Applier
 	switch ptr.Deref(clusterConfig.Addons.CCM.Strategy, "") {
 	case v1alpha1.AddonStrategyHelmAddon:
-		helmChart, err := a.helmChartInfoGetter.For(ctx, log, config.AWSCCM)
+		helmChart, err := a.helmChartInfoGetter.For(ctx, log, cluster, config.AWSCCM)
 		if err != nil {
 			return fmt.Errorf("failed to get configuration to create helm addon: %w", err)
 		}

pkg/handlers/generic/lifecycle/ccm/nutanix/handler.go

@@ -115,7 +115,7 @@ func (p *provider) Apply(
 		}
 	}
 
-	helmChart, err := p.helmChartInfoGetter.For(ctx, log, config.NutanixCCM)
+	helmChart, err := p.helmChartInfoGetter.For(ctx, log, cluster, config.NutanixCCM)
 	if err != nil {
 		return fmt.Errorf("failed to get values for nutanix-ccm-config: %w", err)
 	}

pkg/handlers/generic/lifecycle/clusterautoscaler/handler.go

@@ -146,6 +146,7 @@ func (n *DefaultClusterAutoscaler) apply(
 		helmChart, err := n.helmChartInfoGetter.For(
 			ctx,
 			log,
+			cluster,
 			config.Autoscaler,
 		)
 		if err != nil {

pkg/handlers/generic/lifecycle/cni/calico/handler.go

@@ -154,7 +154,7 @@ func (c *CalicoCNI) apply(
 	case v1alpha1.AddonStrategyHelmAddon:
 		// this is tigera and not calico because we deploy calico via operataor
 		log.Info("fetching settings for tigera-operator-config")
-		helmChart, err := c.helmChartInfoGetter.For(ctx, log, config.Tigera)
+		helmChart, err := c.helmChartInfoGetter.For(ctx, log, cluster, config.Tigera)
 		if err != nil {
 			log.Error(
 				err,

pkg/handlers/generic/lifecycle/cni/cilium/handler.go

@@ -161,7 +161,7 @@ func (c *CiliumCNI) apply(
 			client: c.client,
 		}
 	case v1alpha1.AddonStrategyHelmAddon:
-		helmChart, err := c.helmChartInfoGetter.For(ctx, log, config.Cilium)
+		helmChart, err := c.helmChartInfoGetter.For(ctx, log, cluster, config.Cilium)
 		if err != nil {
 			log.Error(
 				err,

pkg/handlers/generic/lifecycle/config/cm.go

@@ -5,13 +5,18 @@ package config
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/go-logr/logr"
 	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
 )
 
 type Component string
@@ -75,13 +80,28 @@ func (h *HelmChartGetter) get(
 	return cm, err
 }
 
-func (h *HelmChartGetter) For(
+type HelmChartInfoNotFoundError struct {
+	Component Component
+	Name      string
+	Namespace string
+}
+
+func (e HelmChartInfoNotFoundError) Error() string {
+	return fmt.Sprintf(
+		"Helm Chart component %q not found in configmap %s/%s",
+		e.Component,
+		e.Namespace,
+		e.Name,
+	)
+}
+
+func (h *HelmChartGetter) getInfoFor(
 	ctx context.Context,
 	log logr.Logger,
 	name Component,
 ) (*HelmChart, error) {
 	log.Info(
-		fmt.Sprintf("Fetching HelmChart info for %q from configmap %s/%s",
+		fmt.Sprintf("fetching HelmChart info for %q from configmap %s/%s",
 			string(name),
 			h.cmNamespace,
 			h.cmName),
@@ -92,14 +112,59 @@ func (h *HelmChartGetter) For(
 	}
 	d, ok := cm.Data[string(name)]
 	if !ok {
-		return nil, fmt.Errorf(
-			"did not find key %q in configmap %s/%s",
+		return nil, HelmChartInfoNotFoundError{
 			name,
 			h.cmNamespace,
 			h.cmName,
-		)
+		}
 	}
 	var settings HelmChart
 	err = yaml.Unmarshal([]byte(d), &settings)
 	return &settings, err
 }
+
+// For returns the HelmChart info for the given component from the configmap referenced in the cluster variables.
+// It first checks the configmap referenced in the cluster variables.
+// If not found, it returns the HelmChart info from the default configmap.
+func (h *HelmChartGetter) For(
+	ctx context.Context,
+	log logr.Logger,
+	cluster *clusterv1.Cluster,
+	name Component,
+) (*HelmChart, error) {
+	varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables)
+
+	helmChartConfig, err := variables.Get[v1alpha1.HelmChartConfig](
+		varMap,
+		v1alpha1.ClusterConfigVariableName,
+		"addons", "helmChartConfig")
+	if err != nil {
+		if variables.IsNotFoundError(err) {
+			// HelmChartConfig is not defined in the cluster. Get the HelmChart from the default configmap.
+			return h.getInfoFor(ctx, log, name)
+		}
+		return nil, err
+	}
+	// helmChartConfig.ConfigMapRef or helmChartConfig.ConfigMapRef.Name will not be nil
+	// because it will be validated by the schema validation.
+	cmNameFromVariables := helmChartConfig.ConfigMapRef.Name
+	clusterNamespace := cluster.Namespace
+	hcGetter := NewHelmChartGetterFromConfigMap(cmNameFromVariables, clusterNamespace, h.cl)
+	overrideHelmChart, err := hcGetter.getInfoFor(ctx, log, name)
+	if err != nil {
+		if errors.As(err, &HelmChartInfoNotFoundError{}) {
+			// HelmChart is not defined in the custom configmap. Get the HelmChart from the default configmap.
+			return h.getInfoFor(ctx, log, name)
+		}
+		return nil, err
+	}
+	log.Info(
+		fmt.Sprintf(
+			"using HelmChart info for %q from custom configmap  %s/%s referenced in cluster variables",
+			string(name),
+			clusterNamespace,
+			cmNameFromVariables,
+		),
+	)
+	return overrideHelmChart, nil
+}

pkg/handlers/generic/lifecycle/cosi/handler.go

@@ -144,7 +144,7 @@ func (n *DefaultCOSIController) apply(
 	var strategy addons.Applier
 	switch ptr.Deref(cosiVar.Strategy, "") {
 	case v1alpha1.AddonStrategyHelmAddon:
-		helmChart, err := n.helmChartInfoGetter.For(ctx, log, config.COSIController)
+		helmChart, err := n.helmChartInfoGetter.For(ctx, log, cluster, config.COSIController)
 		if err != nil {
 			log.Error(
 				err,

pkg/handlers/generic/lifecycle/csi/awsebs/handler.go

@@ -82,7 +82,7 @@ func (a *AWSEBS) Apply(
 	var strategy addons.Applier
 	switch ptr.Deref(provider.Strategy, "") {
 	case v1alpha1.AddonStrategyHelmAddon:
-		helmChart, err := a.helmChartInfoGetter.For(ctx, log, config.AWSEBSCSI)
+		helmChart, err := a.helmChartInfoGetter.For(ctx, log, cluster, config.AWSEBSCSI)
 		if err != nil {
 			return fmt.Errorf("failed to get configuration to create helm addon: %w", err)
 		}

pkg/handlers/generic/lifecycle/csi/localpath/handler.go

@@ -78,7 +78,12 @@ func (l *LocalPathProvisionerCSI) Apply(
 	var strategy addons.Applier
 	switch ptr.Deref(provider.Strategy, "") {
 	case v1alpha1.AddonStrategyHelmAddon:
-		helmChart, err := l.helmChartInfoGetter.For(ctx, log, config.LocalPathProvisionerCSI)
+		helmChart, err := l.helmChartInfoGetter.For(
+			ctx,
+			log,
+			cluster,
+			config.LocalPathProvisionerCSI,
+		)
 		if err != nil {
 			return fmt.Errorf("failed to get configuration to create helm addon: %w", err)
 		}