Merge pull request #8785 from alexander-demicev/capdloadbalancer

✨ Allow adding custom HA proxy config for CAPD load balancer

Author: k8s-ci-robot

test/infrastructure/docker/api/v1alpha3/conversion.go

@@ -45,6 +45,10 @@ func (src *DockerCluster) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Spec.LoadBalancer.ImageTag = restored.Spec.LoadBalancer.ImageTag
 	}
 
+	if restored.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef != nil {
+		dst.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef = restored.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef
+	}
+
 	return nil
 }
 

test/infrastructure/docker/api/v1alpha4/conversion.go

@@ -27,13 +27,35 @@ import (
 func (src *DockerCluster) ConvertTo(dstRaw conversion.Hub) error {
 	dst := dstRaw.(*infrav1.DockerCluster)
 
-	return Convert_v1alpha4_DockerCluster_To_v1beta1_DockerCluster(src, dst, nil)
+	if err := Convert_v1alpha4_DockerCluster_To_v1beta1_DockerCluster(src, dst, nil); err != nil {
+		return err
+	}
+
+	// Manually restore data.
+	restored := &infrav1.DockerCluster{}
+	if ok, err := utilconversion.UnmarshalData(src, restored); err != nil || !ok {
+		return err
+	}
+
+	if restored.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef != nil {
+		dst.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef = restored.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef
+	}
+
+	return nil
 }
 
 func (dst *DockerCluster) ConvertFrom(srcRaw conversion.Hub) error {
 	src := srcRaw.(*infrav1.DockerCluster)
 
-	return Convert_v1beta1_DockerCluster_To_v1alpha4_DockerCluster(src, dst, nil)
+	if err := Convert_v1beta1_DockerCluster_To_v1alpha4_DockerCluster(src, dst, nil); err != nil {
+		return err
+	}
+
+	if err := utilconversion.MarshalData(src, dst); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (src *DockerClusterList) ConvertTo(dstRaw conversion.Hub) error {
@@ -63,6 +85,10 @@ func (src *DockerClusterTemplate) ConvertTo(dstRaw conversion.Hub) error {
 
 	dst.Spec.Template.ObjectMeta = restored.Spec.Template.ObjectMeta
 
+	if restored.Spec.Template.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef != nil {
+		dst.Spec.Template.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef = restored.Spec.Template.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef
+	}
+
 	return nil
 }
 
@@ -171,3 +197,7 @@ func Convert_v1beta1_DockerMachineTemplateResource_To_v1alpha4_DockerMachineTemp
 	// NOTE: custom conversion func is required because spec.template.metadata has been added in v1beta1.
 	return autoConvert_v1beta1_DockerMachineTemplateResource_To_v1alpha4_DockerMachineTemplateResource(in, out, s)
 }
+
+func Convert_v1beta1_DockerLoadBalancer_To_v1alpha4_DockerLoadBalancer(in *infrav1.DockerLoadBalancer, out *DockerLoadBalancer, s apiconversion.Scope) error {
+	return autoConvert_v1beta1_DockerLoadBalancer_To_v1alpha4_DockerLoadBalancer(in, out, s)
+}

test/infrastructure/docker/api/v1alpha4/zz_generated.conversion.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1beta1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -53,6 +54,17 @@ type DockerClusterSpec struct {
 type DockerLoadBalancer struct {
 	// ImageMeta allows customizing the image used for the cluster load balancer.
 	ImageMeta `json:",inline"`
+
+	// CustomHAProxyConfigTemplateRef allows you to replace the default HAProxy config file.
+	// This field is a reference to a config map that contains the configuration template. The key of the config map should be equal to 'value'.
+	// The content of the config map will be processed and will replace the default HAProxy config file. Please use it with caution, as there are
+	// no checks to ensure the validity of the configuration. This template will support the following variables that will be passed by the controller:
+	// $IPv6 (bool) indicates if the cluster is IPv6, $FrontendControlPlanePort (string) indicates the frontend control plane port,
+	// $BackendControlPlanePort (string) indicates the backend control plane port, $BackendServers (map[string]string) indicates the backend server
+	// where the key is the server name and the value is the address. This map is dynamic and is updated every time a new control plane
+	// node is added or removed. The template will also support the JoinHostPort function to join the host and port of the backend server.
+	// +optional
+	CustomHAProxyConfigTemplateRef *corev1.LocalObjectReference `json:"customHAProxyConfigTemplateRef,omitempty"`
 }
 
 // ImageMeta allows customizing the image used for components that are not

test/infrastructure/docker/api/v1beta1/dockercluster_types.go

@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -167,11 +168,11 @@ func (r *DockerMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	// Handle deleted machines
 	if !dockerMachine.ObjectMeta.DeletionTimestamp.IsZero() {
-		return ctrl.Result{}, r.reconcileDelete(ctx, machine, dockerMachine, externalMachine, externalLoadBalancer)
+		return ctrl.Result{}, r.reconcileDelete(ctx, dockerCluster, machine, dockerMachine, externalMachine, externalLoadBalancer)
 	}
 
 	// Handle non-deleted machines
-	res, err := r.reconcileNormal(ctx, cluster, machine, dockerMachine, externalMachine, externalLoadBalancer)
+	res, err := r.reconcileNormal(ctx, cluster, dockerCluster, machine, dockerMachine, externalMachine, externalLoadBalancer)
 	// Requeue if the reconcile failed because the ClusterCacheTracker was locked for
 	// the current cluster because of concurrent access.
 	if errors.Is(err, remote.ErrClusterLocked) {
@@ -204,7 +205,7 @@ func patchDockerMachine(ctx context.Context, patchHelper *patch.Helper, dockerMa
 	)
 }
 
-func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) (res ctrl.Result, retErr error) {
+func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *clusterv1.Cluster, dockerCluster *infrav1.DockerCluster, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) (res ctrl.Result, retErr error) {
 	log := ctrl.LoggerFrom(ctx)
 
 	// Check if the infrastructure is ready, otherwise return and wait for the cluster object to be updated
@@ -271,7 +272,11 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *
 	// we should only do this once, as reconfiguration more or less ensures
 	// node ref setting fails
 	if util.IsControlPlaneMachine(machine) && !dockerMachine.Status.LoadBalancerConfigured {
-		if err := externalLoadBalancer.UpdateConfiguration(ctx); err != nil {
+		unsafeLoadBalancerConfigTemplate, err := r.getUnsafeLoadBalancerConfigTemplate(ctx, dockerCluster)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to retrieve HAProxy configuration from CustomHAProxyConfigTemplateRef")
+		}
+		if err := externalLoadBalancer.UpdateConfiguration(ctx, unsafeLoadBalancerConfigTemplate); err != nil {
 			return ctrl.Result{}, errors.Wrap(err, "failed to update DockerCluster.loadbalancer configuration")
 		}
 		dockerMachine.Status.LoadBalancerConfigured = true
@@ -390,7 +395,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *
 	return ctrl.Result{}, nil
 }
 
-func (r *DockerMachineReconciler) reconcileDelete(ctx context.Context, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) error {
+func (r *DockerMachineReconciler) reconcileDelete(ctx context.Context, dockerCluster *infrav1.DockerCluster, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) error {
 	// Set the ContainerProvisionedCondition reporting delete is started, and issue a patch in order to make
 	// this visible to the users.
 	// NB. The operation in docker is fast, so there is the chance the user will not notice the status change;
@@ -411,7 +416,11 @@ func (r *DockerMachineReconciler) reconcileDelete(ctx context.Context, machine *
 
 	// if the deleted machine is a control-plane node, remove it from the load balancer configuration;
 	if util.IsControlPlaneMachine(machine) {
-		if err := externalLoadBalancer.UpdateConfiguration(ctx); err != nil {
+		unsafeLoadBalancerConfigTemplate, err := r.getUnsafeLoadBalancerConfigTemplate(ctx, dockerCluster)
+		if err != nil {
+			return errors.Wrap(err, "failed to retrieve HAProxy configuration from CustomHAProxyConfigTemplateRef")
+		}
+		if err := externalLoadBalancer.UpdateConfiguration(ctx, unsafeLoadBalancerConfigTemplate); err != nil {
 			return errors.Wrap(err, "failed to update DockerCluster.loadbalancer configuration")
 		}
 	}
@@ -510,6 +519,25 @@ func (r *DockerMachineReconciler) getBootstrapData(ctx context.Context, machine
 	return base64.StdEncoding.EncodeToString(value), bootstrapv1.Format(format), nil
 }
 
+func (r *DockerMachineReconciler) getUnsafeLoadBalancerConfigTemplate(ctx context.Context, dockerCluster *infrav1.DockerCluster) (string, error) {
+	if dockerCluster.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef == nil {
+		return "", nil
+	}
+	var cm *corev1.ConfigMap
+	key := types.NamespacedName{
+		Name:      dockerCluster.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef.Name,
+		Namespace: dockerCluster.Namespace,
+	}
+	if err := r.Get(ctx, key, cm); err != nil {
+		return "", errors.Wrapf(err, "failed to retrieve custom HAProxy configuration ConfigMap %s", key)
+	}
+	template, ok := cm.Data["value"]
+	if !ok {
+		return "", fmt.Errorf("expected key \"value\" to exist in ConfigMap %s", key)
+	}
+	return template, nil
+}
+
 // setMachineAddress gets the address from the container corresponding to a docker node and sets it on the Machine object.
 func setMachineAddress(ctx context.Context, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine) error {
 	machineAddresses, err := externalMachine.Address(ctx)