diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md index 795babc41..20bf4439e 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/README.md +++ b/charts/cluster-api-runtime-extensions-nutanix/README.md @@ -78,6 +78,8 @@ A Helm chart for cluster-api-runtime-extensions-nutanix | hooks.cni.cilium.crsStrategy.defaultCiliumConfigMap.name | string | `"cilium"` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.cni.cilium.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cilium-cni-helm-values-template"` | | +| hooks.cni.multus.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | +| hooks.cni.multus.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-multus-values-template"` | | | hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | | hooks.cosi.controller.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-cosi-controller-helm-values-template"` | | | hooks.csi.aws-ebs.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` | | diff --git a/charts/cluster-api-runtime-extensions-nutanix/addons/cni/multus/values-template.yaml b/charts/cluster-api-runtime-extensions-nutanix/addons/cni/multus/values-template.yaml new file mode 100644 index 000000000..2fc901b0b --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/addons/cni/multus/values-template.yaml @@ -0,0 +1,18 @@ +# Multus daemon configuration overrides +daemonConfig: + readinessIndicatorFile: "{{ .ReadinessSocketPath }}" + +{{- if .ReadinessSocketPath }} +# Volumes for CNI readiness socket +volumes: + - name: cni-readiness-sock + hostPath: + path: "{{ .ReadinessSocketPath }}" + type: Socket + +# Volume mounts for CNI readiness socket +volumeMounts: + - name: cni-readiness-sock + mountPath: "{{ .ReadinessSocketPath }}" + readOnly: true +{{- end }} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/cni/multus/manifests/helm-addon-installation.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/multus/manifests/helm-addon-installation.yaml new file mode 100644 index 000000000..3f91825e8 --- /dev/null +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/cni/multus/manifests/helm-addon-installation.yaml @@ -0,0 +1,12 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.hooks.cni.multus.helmAddonStrategy.defaultValueTemplateConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: '{{ .Values.hooks.cni.multus.helmAddonStrategy.defaultValueTemplateConfigMap.name }}' +data: + values.yaml: |- + {{- .Files.Get "addons/cni/multus/values-template.yaml" | nindent 4 }} +{{- end -}} diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml index 2939fd87e..7b30e8ec6 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml @@ -43,6 +43,10 @@ data: ChartName: metallb ChartVersion: 0.15.2 RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://metallb.github.io/metallb{{ end }}' + multus: | + ChartName: multus + ChartVersion: 0.1.0 + RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://mesosphere.github.io/charts/stable/{{ end }}' nfd: | ChartName: node-feature-discovery ChartVersion: 0.18.1 diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json index 8b22ed15a..08da00ae0 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json +++ b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json @@ -381,6 +381,27 @@ } } } + }, + "multus": { + "type": "object", + "properties": { + "helmAddonStrategy": { + "type": "object", + "properties": { + "defaultValueTemplateConfigMap": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } + } + } + } + } } } }, diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml index 080076816..ac2571996 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/values.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/values.yaml @@ -43,6 +43,11 @@ hooks: defaultValueTemplateConfigMap: create: true name: default-cilium-cni-helm-values-template + multus: + helmAddonStrategy: + defaultValueTemplateConfigMap: + create: true + name: default-multus-values-template csi: nutanix: helmAddonStrategy: diff --git a/hack/addons/helm-chart-bundler/repos.yaml b/hack/addons/helm-chart-bundler/repos.yaml index 3038aefc4..70c9d767f 100644 --- a/hack/addons/helm-chart-bundler/repos.yaml +++ b/hack/addons/helm-chart-bundler/repos.yaml @@ -51,6 +51,11 @@ repositories: charts: metallb: - 0.15.2 + multus: + repoURL: https://mesosphere.github.io/charts/stable/ + charts: + multus: + - 0.1.0 node-feature-discovery: repoURL: https://kubernetes-sigs.github.io/node-feature-discovery/charts charts: diff --git a/hack/addons/kustomize/multus/kustomization.yaml.tmpl b/hack/addons/kustomize/multus/kustomization.yaml.tmpl new file mode 100644 index 000000000..0fdf7c72e --- /dev/null +++ b/hack/addons/kustomize/multus/kustomization.yaml.tmpl @@ -0,0 +1,20 @@ +# Copyright 2024 Nutanix. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +metadata: + name: multus + +sortOptions: + order: fifo + +helmCharts: +- name: multus + namespace: kube-system + repo: https://mesosphere.github.io/charts/stable/ + releaseName: multus + version: 0.1.0 + includeCRDs: true + skipTests: true diff --git a/pkg/handlers/lifecycle/cni/multus/doc.go b/pkg/handlers/lifecycle/cni/multus/doc.go new file mode 100644 index 000000000..51e2b0e2f --- /dev/null +++ b/pkg/handlers/lifecycle/cni/multus/doc.go @@ -0,0 +1,24 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package multus provides a standalone lifecycle handler for Multus CNI that automatically +// deploys Multus when: +// - The cluster is on EKS cloud provider +// - A supported CNI provider is configured (Cilium or Calico) +// +// MultusHandler implements the cluster lifecycle hooks and: +// - Detects the cloud provider from the cluster infrastructure +// - Reads CNI configuration from cluster variables +// - Gets the readiness socket path for the configured CNI (via cni.ReadinessSocketPath) +// - Automatically deploys Multus with socket-based configuration +// +// helmAddonStrategy is the internal strategy that handles: +// - Templating Helm values with the CNI socket path +// - Deploying Multus using HelmAddon strategy with Go template-based values +// +// Multus relies on the readinessIndicatorFile configuration to wait for the primary CNI +// to be ready, eliminating the need for explicit wait logic in the strategy. +// +// This package does NOT expose Multus in the API - it's an internal addon +// that deploys automatically based on cloud provider and CNI selection. +package multus diff --git a/pkg/handlers/lifecycle/cni/multus/handler.go b/pkg/handlers/lifecycle/cni/multus/handler.go new file mode 100644 index 000000000..88e5dcec0 --- /dev/null +++ b/pkg/handlers/lifecycle/cni/multus/handler.go @@ -0,0 +1,181 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package multus + +import ( + "context" + "fmt" + + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + commonhandlers "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/lifecycle" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" + capiutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/utils" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/addons" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/config" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/options" +) + +const ( + defaultMultusReleaseName = "multus" + defaultMultusNamespace = metav1.NamespaceSystem +) + +type MultusConfig struct { + *options.GlobalOptions + + helmAddonConfig *addons.HelmAddonConfig +} + +func NewMultusConfig(globalOptions *options.GlobalOptions) *MultusConfig { + return &MultusConfig{ + GlobalOptions: globalOptions, + helmAddonConfig: addons.NewHelmAddonConfig( + "default-multus-values-template", + defaultMultusNamespace, + defaultMultusReleaseName, + ), + } +} + +func (m *MultusConfig) AddFlags(prefix string, flags *pflag.FlagSet) { + m.helmAddonConfig.AddFlags(prefix+".helm-addon", flags) +} + +type MultusHandler struct { + client ctrlclient.Client + config *MultusConfig + helmChartInfoGetter *config.HelmChartGetter +} + +var ( + _ commonhandlers.Named = &MultusHandler{} + _ lifecycle.AfterControlPlaneInitialized = &MultusHandler{} + _ lifecycle.BeforeClusterUpgrade = &MultusHandler{} +) + +func New( + c ctrlclient.Client, + cfg *MultusConfig, + helmChartInfoGetter *config.HelmChartGetter, +) *MultusHandler { + return &MultusHandler{ + client: c, + config: cfg, + helmChartInfoGetter: helmChartInfoGetter, + } +} + +func (m *MultusHandler) Name() string { + return "MultusHandler" +} + +func (m *MultusHandler) AfterControlPlaneInitialized( + ctx context.Context, + req *runtimehooksv1.AfterControlPlaneInitializedRequest, + resp *runtimehooksv1.AfterControlPlaneInitializedResponse, +) { + commonResponse := &runtimehooksv1.CommonResponse{} + m.apply(ctx, &req.Cluster, commonResponse) + resp.Status = commonResponse.GetStatus() + resp.Message = commonResponse.GetMessage() +} + +func (m *MultusHandler) BeforeClusterUpgrade( + ctx context.Context, + req *runtimehooksv1.BeforeClusterUpgradeRequest, + resp *runtimehooksv1.BeforeClusterUpgradeResponse, +) { + commonResponse := &runtimehooksv1.CommonResponse{} + m.apply(ctx, &req.Cluster, commonResponse) + resp.Status = commonResponse.GetStatus() + resp.Message = commonResponse.GetMessage() +} + +func (m *MultusHandler) apply( + ctx context.Context, + cluster *clusterv1.Cluster, + resp *runtimehooksv1.CommonResponse, +) { + clusterKey := ctrlclient.ObjectKeyFromObject(cluster) + + log := ctrl.LoggerFrom(ctx).WithValues( + "cluster", + clusterKey, + ) + + // Check if Multus is supported for this cloud provider + provider := capiutils.GetProvider(cluster) + if provider != "eks" { + log.V(5).Info( + "Multus is not supported for this cloud provider. Skipping Multus deployment.", + ) + return + } + + log.Info(fmt.Sprintf("Cluster is %s. Checking CNI configuration for Multus deployment.", provider)) + + // Read CNI configuration to detect which CNI is deployed + varMap := variables.ClusterVariablesToVariablesMap(cluster.Spec.Topology.Variables) + + cniVar, err := variables.Get[v1alpha1.CNI]( + varMap, + v1alpha1.ClusterConfigVariableName, + []string{"addons", v1alpha1.CNIVariableName}...) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5).Info("No CNI specified in cluster config. Skipping Multus deployment.") + return + } + log.Error(err, "failed to read CNI configuration from cluster definition") + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage(fmt.Sprintf("failed to read CNI configuration: %v", err)) + return + } + + log.Info(fmt.Sprintf("Auto-deploying Multus for %s cluster with %s CNI", provider, cniVar.Provider)) + + // Get helm chart configuration + helmChart, err := m.helmChartInfoGetter.For(ctx, log, config.Multus) + if err != nil { + log.Error( + err, + "failed to get configmap with helm settings", + ) + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage( + fmt.Sprintf("failed to get configuration to create helm addon: %v", + err, + ), + ) + return + } + + // Create and apply helm addon using existing addons package + targetNamespace := m.config.DefaultsNamespace() + strategy := addons.NewHelmAddonApplier( + m.config.helmAddonConfig, + m.client, + helmChart, + ). + WithValueTemplater(templateValuesFunc(cniVar)). + WithDefaultWaiter() + + if err := strategy.Apply(ctx, cluster, targetNamespace, log); err != nil { + log.Error(err, "failed to deploy Multus") + resp.SetStatus(runtimehooksv1.ResponseStatusFailure) + resp.SetMessage(fmt.Sprintf("failed to deploy Multus: %v", err)) + return + } + + resp.SetStatus(runtimehooksv1.ResponseStatusSuccess) + resp.SetMessage("Multus deployed successfully") +} diff --git a/pkg/handlers/lifecycle/cni/multus/template.go b/pkg/handlers/lifecycle/cni/multus/template.go new file mode 100644 index 000000000..bfb4a48e2 --- /dev/null +++ b/pkg/handlers/lifecycle/cni/multus/template.go @@ -0,0 +1,49 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package multus + +import ( + "bytes" + "fmt" + "text/template" + + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/cni" +) + +// templateValuesFunc returns a template function that parses the Multus values template +// and replaces socket-related placeholders with the readiness socket path. +// It looks up the socket path internally based on the CNI provider. +func templateValuesFunc(cniVar v1alpha1.CNI) func(*clusterv1.Cluster, string) (string, error) { + return func(_ *clusterv1.Cluster, valuesTemplate string) (string, error) { + // Look up the readiness socket path for the CNI provider + readinessSocketPath, err := cni.ReadinessSocketPath(cniVar.Provider) + if err != nil { + return "", fmt.Errorf("failed to get readiness socket path for CNI provider %s: %w", cniVar.Provider, err) + } + + t, err := template.New("").Parse(valuesTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse Multus values template: %w", err) + } + + type input struct { + ReadinessSocketPath string + } + + templateInput := input{ + ReadinessSocketPath: readinessSocketPath, + } + + var b bytes.Buffer + err = t.Execute(&b, templateInput) + if err != nil { + return "", fmt.Errorf("failed to template Multus values: %w", err) + } + + return b.String(), nil + } +} diff --git a/pkg/handlers/lifecycle/cni/socket.go b/pkg/handlers/lifecycle/cni/socket.go new file mode 100644 index 000000000..30c410425 --- /dev/null +++ b/pkg/handlers/lifecycle/cni/socket.go @@ -0,0 +1,24 @@ +// Copyright 2024 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cni + +import ( + "fmt" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" +) + +// ReadinessSocketPath returns the readiness socket path for the given CNI provider. +// The socket path is used by Multus to wait for the primary CNI to be ready. +// Returns an empty string and error for unsupported providers. +func ReadinessSocketPath(provider string) (string, error) { + switch provider { + case v1alpha1.CNIProviderCilium: + return "/run/cilium/cilium.sock", nil + case v1alpha1.CNIProviderCalico: + return "/var/run/calico/cni-server.sock", nil + default: + return "", fmt.Errorf("could not determine CNI socket, unsupported provider: %s", provider) + } +} diff --git a/pkg/handlers/lifecycle/config/cm.go b/pkg/handlers/lifecycle/config/cm.go index 73a714156..306aab941 100644 --- a/pkg/handlers/lifecycle/config/cm.go +++ b/pkg/handlers/lifecycle/config/cm.go @@ -32,6 +32,7 @@ const ( COSIController Component = "cosi-controller" CNCFDistributionRegistry Component = "cncf-distribution-registry" RegistrySyncer Component = "registry-syncer" + Multus Component = "multus" ) type HelmChartGetter struct { diff --git a/pkg/handlers/lifecycle/handlers.go b/pkg/handlers/lifecycle/handlers.go index 03f6808b6..aae8bb5f5 100644 --- a/pkg/handlers/lifecycle/handlers.go +++ b/pkg/handlers/lifecycle/handlers.go @@ -17,6 +17,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/clusterautoscaler" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/cni/calico" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/cni/cilium" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/cni/multus" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/config" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/cosi" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/lifecycle/csi" @@ -38,6 +39,7 @@ type Handlers struct { globalOptions *options.GlobalOptions calicoCNIConfig *calico.CNIConfig ciliumCNIConfig *cilium.CNIConfig + multusConfig *multus.MultusConfig nfdConfig *nfd.Config clusterAutoscalerConfig *clusterautoscaler.Config ebsConfig *awsebs.Config @@ -61,6 +63,7 @@ func New( GlobalOptions: globalOptions, }, ciliumCNIConfig: &cilium.CNIConfig{GlobalOptions: globalOptions}, + multusConfig: multus.NewMultusConfig(globalOptions), nfdConfig: nfd.NewConfig(globalOptions), clusterAutoscalerConfig: &clusterautoscaler.Config{GlobalOptions: globalOptions}, ebsConfig: awsebs.NewConfig(globalOptions), @@ -124,6 +127,7 @@ func (h *Handlers) AllHandlers(mgr manager.Manager) []handlers.Named { allHandlers := []handlers.Named{ calico.New(mgr.GetClient(), h.calicoCNIConfig, helmChartInfoGetter), cilium.New(mgr.GetClient(), h.ciliumCNIConfig, helmChartInfoGetter), + multus.New(mgr.GetClient(), h.multusConfig, helmChartInfoGetter), ccm.New(mgr.GetClient(), ccmHandlers), nfd.New(mgr.GetClient(), h.nfdConfig, helmChartInfoGetter), clusterautoscaler.New(mgr.GetClient(), h.clusterAutoscalerConfig, helmChartInfoGetter),