Replace HA proxy config with custom one if specified

Author: alexander-demicev

test/infrastructure/docker/api/v1beta1/dockercluster_types.go

@@ -55,7 +55,7 @@ type DockerLoadBalancer struct {
 	// ImageMeta allows customizing the image used for the cluster load balancer.
 	ImageMeta `json:",inline"`
 
-	// UnsafeHAProxyConfigTemplateRef allows you to replace the HAProxy config file.
+	// CustomHAProxyConfigTemplateRef allows you to replace the default HAProxy config file.
 	// This field is a reference to a config map that contains the configuration template. The key of the config map should be equal to 'value'.
 	// The content of the config map will be processed and will replace the default HAProxy config file. Please use it with caution, as there are
 	// no checks to ensure the validity of the configuration. This template will support the following variables that will be passed by the controller:
@@ -64,7 +64,7 @@ type DockerLoadBalancer struct {
 	// where the key is the server name and the value is the address. This map is dynamic and is updated every time a new control plane
 	// node is added or removed. The template will also support the JoinHostPort function to join the host and port of the backend server.
 	// +optional
-	CustomHAProxyConfigTemplateRef *corev1.LocalObjectReference `json:"unsafeHAProxyConfigTemplateRef,omitempty"`
+	CustomHAProxyConfigTemplateRef *corev1.LocalObjectReference `json:"customHAProxyConfigTemplateRef,omitempty"`
 }
 
 // ImageMeta allows customizing the image used for components that are not

test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockerclusters.yaml

@@ -25,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
@@ -167,11 +168,11 @@ func (r *DockerMachineReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	// Handle deleted machines
 	if !dockerMachine.ObjectMeta.DeletionTimestamp.IsZero() {
-		return ctrl.Result{}, r.reconcileDelete(ctx, machine, dockerMachine, externalMachine, externalLoadBalancer)
+		return ctrl.Result{}, r.reconcileDelete(ctx, dockerCluster, machine, dockerMachine, externalMachine, externalLoadBalancer)
 	}
 
 	// Handle non-deleted machines
-	res, err := r.reconcileNormal(ctx, cluster, machine, dockerMachine, externalMachine, externalLoadBalancer)
+	res, err := r.reconcileNormal(ctx, cluster, dockerCluster, machine, dockerMachine, externalMachine, externalLoadBalancer)
 	// Requeue if the reconcile failed because the ClusterCacheTracker was locked for
 	// the current cluster because of concurrent access.
 	if errors.Is(err, remote.ErrClusterLocked) {
@@ -204,7 +205,7 @@ func patchDockerMachine(ctx context.Context, patchHelper *patch.Helper, dockerMa
 	)
 }
 
-func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *clusterv1.Cluster, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) (res ctrl.Result, retErr error) {
+func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *clusterv1.Cluster, dockerCluster *infrav1.DockerCluster, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) (res ctrl.Result, retErr error) {
 	log := ctrl.LoggerFrom(ctx)
 
 	// Check if the infrastructure is ready, otherwise return and wait for the cluster object to be updated
@@ -271,7 +272,11 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *
 	// we should only do this once, as reconfiguration more or less ensures
 	// node ref setting fails
 	if util.IsControlPlaneMachine(machine) && !dockerMachine.Status.LoadBalancerConfigured {
-		if err := externalLoadBalancer.UpdateConfiguration(ctx); err != nil {
+		unsafeLoadBalancerConfigTemplate, err := r.getUnsafeLoadBalancerConfigTemplate(ctx, dockerCluster)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "failed to retrieve HAProxy configuration from CustomHAProxyConfigTemplateRef")
+		}
+		if err := externalLoadBalancer.UpdateConfiguration(ctx, unsafeLoadBalancerConfigTemplate); err != nil {
 			return ctrl.Result{}, errors.Wrap(err, "failed to update DockerCluster.loadbalancer configuration")
 		}
 		dockerMachine.Status.LoadBalancerConfigured = true
@@ -390,7 +395,7 @@ func (r *DockerMachineReconciler) reconcileNormal(ctx context.Context, cluster *
 	return ctrl.Result{}, nil
 }
 
-func (r *DockerMachineReconciler) reconcileDelete(ctx context.Context, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) error {
+func (r *DockerMachineReconciler) reconcileDelete(ctx context.Context, dockerCluster *infrav1.DockerCluster, machine *clusterv1.Machine, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine, externalLoadBalancer *docker.LoadBalancer) error {
 	// Set the ContainerProvisionedCondition reporting delete is started, and issue a patch in order to make
 	// this visible to the users.
 	// NB. The operation in docker is fast, so there is the chance the user will not notice the status change;
@@ -411,7 +416,11 @@ func (r *DockerMachineReconciler) reconcileDelete(ctx context.Context, machine *
 
 	// if the deleted machine is a control-plane node, remove it from the load balancer configuration;
 	if util.IsControlPlaneMachine(machine) {
-		if err := externalLoadBalancer.UpdateConfiguration(ctx); err != nil {
+		unsafeLoadBalancerConfigTemplate, err := r.getUnsafeLoadBalancerConfigTemplate(ctx, dockerCluster)
+		if err != nil {
+			return errors.Wrap(err, "failed to retrieve HAProxy configuration from CustomHAProxyConfigTemplateRef")
+		}
+		if err := externalLoadBalancer.UpdateConfiguration(ctx, unsafeLoadBalancerConfigTemplate); err != nil {
 			return errors.Wrap(err, "failed to update DockerCluster.loadbalancer configuration")
 		}
 	}
@@ -510,6 +519,25 @@ func (r *DockerMachineReconciler) getBootstrapData(ctx context.Context, machine
 	return base64.StdEncoding.EncodeToString(value), bootstrapv1.Format(format), nil
 }
 
+func (r *DockerMachineReconciler) getUnsafeLoadBalancerConfigTemplate(ctx context.Context, dockerCluster *infrav1.DockerCluster) (string, error) {
+	if dockerCluster.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef == nil {
+		return "", nil
+	}
+	var cm *corev1.ConfigMap
+	key := types.NamespacedName{
+		Name:      dockerCluster.Spec.LoadBalancer.CustomHAProxyConfigTemplateRef.Name,
+		Namespace: dockerCluster.Namespace,
+	}
+	if err := r.Get(ctx, key, cm); err != nil {
+		return "", errors.Wrapf(err, "failed to retrieve custom HAProxy configuration ConfigMap %s", key)
+	}
+	template, ok := cm.Data["value"]
+	if !ok {
+		return "", fmt.Errorf("expected key \"value\" to exist in ConfigMap %s", key)
+	}
+	return template, nil
+}
+
 // setMachineAddress gets the address from the container corresponding to a docker node and sets it on the Machine object.
 func setMachineAddress(ctx context.Context, dockerMachine *infrav1.DockerMachine, externalMachine *docker.Machine) error {
 	machineAddresses, err := externalMachine.Address(ctx)

test/infrastructure/docker/config/crd/bases/infrastructure.cluster.x-k8s.io_dockerclustertemplates.yaml

@@ -19,7 +19,7 @@ package docker
 import (
 	"context"
 	"fmt"
-	"net"
+	"strconv"
 
 	"github.com/pkg/errors"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -38,12 +38,13 @@ type lbCreator interface {
 
 // LoadBalancer manages the load balancer for a specific docker cluster.
 type LoadBalancer struct {
-	name             string
-	image            string
-	container        *types.Node
-	ipFamily         clusterv1.ClusterIPFamily
-	lbCreator        lbCreator
-	controlPlanePort int
+	name                     string
+	image                    string
+	container                *types.Node
+	ipFamily                 clusterv1.ClusterIPFamily
+	lbCreator                lbCreator
+	backendControlPlanePort  string
+	frontendControlPlanePort string
 }
 
 // NewLoadBalancer returns a new helper for managing a docker loadbalancer with a given name.
@@ -72,12 +73,13 @@ func NewLoadBalancer(ctx context.Context, cluster *clusterv1.Cluster, dockerClus
 	image := getLoadBalancerImage(dockerCluster)
 
 	return &LoadBalancer{
-		name:             cluster.Name,
-		image:            image,
-		container:        container,
-		ipFamily:         ipFamily,
-		lbCreator:        &Manager{},
-		controlPlanePort: dockerCluster.Spec.ControlPlaneEndpoint.Port,
+		name:                     cluster.Name,
+		image:                    image,
+		container:                container,
+		ipFamily:                 ipFamily,
+		lbCreator:                &Manager{},
+		frontendControlPlanePort: strconv.Itoa(dockerCluster.Spec.ControlPlaneEndpoint.Port),
+		backendControlPlanePort:  "6443",
 	}, nil
 }
 
@@ -137,7 +139,7 @@ func (s *LoadBalancer) Create(ctx context.Context) error {
 }
 
 // UpdateConfiguration updates the external load balancer configuration with new control plane nodes.
-func (s *LoadBalancer) UpdateConfiguration(ctx context.Context) error {
+func (s *LoadBalancer) UpdateConfiguration(ctx context.Context, unsafeLoadBalancerConfig string) error {
 	log := ctrl.LoggerFrom(ctx)
 
 	if s.container == nil {
@@ -161,17 +163,25 @@ func (s *LoadBalancer) UpdateConfiguration(ctx context.Context) error {
 			return errors.Wrapf(err, "failed to get IP for container %s", n.String())
 		}
 		if s.ipFamily == clusterv1.IPv6IPFamily {
-			backendServers[n.String()] = net.JoinHostPort(controlPlaneIPv6, "6443")
+			backendServers[n.String()] = controlPlaneIPv6
 		} else {
-			backendServers[n.String()] = net.JoinHostPort(controlPlaneIPv4, "6443")
+			backendServers[n.String()] = controlPlaneIPv4
 		}
 	}
 
+	loadBalancerConfigTemplate := loadbalancer.DefaultTemplate
+	if unsafeLoadBalancerConfig != "" {
+		loadBalancerConfigTemplate = unsafeLoadBalancerConfig
+	}
+
 	loadBalancerConfig, err := loadbalancer.Config(&loadbalancer.ConfigData{
-		ControlPlanePort: s.controlPlanePort,
-		BackendServers:   backendServers,
-		IPv6:             s.ipFamily == clusterv1.IPv6IPFamily,
-	})
+		FrontendControlPlanePort: s.frontendControlPlanePort,
+		BackendControlPlanePort:  s.backendControlPlanePort,
+		BackendServers:           backendServers,
+		IPv6:                     s.ipFamily == clusterv1.IPv6IPFamily,
+	},
+		loadBalancerConfigTemplate,
+	)
 	if err != nil {
 		return errors.WithStack(err)
 	}