add metric annotation support to IPPool objects

Author: Felix Wischke (65278)

api/v1alpha1/proxmoxcluster_types.go

@@ -107,6 +107,10 @@ type IPConfigSpec struct {
 	// Gateway
 	// +optional
 	Gateway string `json:"gateway,omitempty"`
+
+	// Metric is the route priority applied to the default gateway
+	// +kubebuilder:default=100
+	Metric *uint32 `json:"metric"`
 }
 
 // SchedulerHints allows to pass the scheduler instructions to (dis)allow over- or enforce underprovisioning of resources.

api/v1alpha1/proxmoxcluster_types_test.go

@@ -83,6 +83,8 @@ func defaultCluster() *ProxmoxCluster {
 			IPv4Config: &IPConfigSpec{
 				Addresses: []string{"10.0.0.0/24"},
 				Prefix:    24,
+				Gateway:   "10.0.0.254",
+				Metric:    func() *uint32 { var a uint32 = 123; return &a }(),
 			},
 			DNSServers: []string{"1.2.3.4"},
 			CloneSpec: &ProxmoxClusterCloneSpec{

api/v1alpha1/zz_generated.deepcopy.go

@@ -598,12 +598,19 @@ spec:
                   gateway:
                     description: Gateway
                     type: string
+                  metric:
+                    default: 100
+                    description: Metric is the route priority applied to the default
+                      gateway
+                    format: int32
+                    type: integer
                   prefix:
                     description: Prefix is the network prefix to use.
                     maximum: 128
                     type: integer
                 required:
                 - addresses
+                - metric
                 - prefix
                 type: object
                 x-kubernetes-validations:
@@ -624,12 +631,19 @@ spec:
                   gateway:
                     description: Gateway
                     type: string
+                  metric:
+                    default: 100
+                    description: Metric is the route priority applied to the default
+                      gateway
+                    format: int32
+                    type: integer
                   prefix:
                     description: Prefix is the network prefix to use.
                     maximum: 128
                     type: integer
                 required:
                 - addresses
+                - metric
                 - prefix
                 type: object
                 x-kubernetes-validations:

config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml

@@ -648,12 +648,19 @@ spec:
                           gateway:
                             description: Gateway
                             type: string
+                          metric:
+                            default: 100
+                            description: Metric is the route priority applied to the
+                              default gateway
+                            format: int32
+                            type: integer
                           prefix:
                             description: Prefix is the network prefix to use.
                             maximum: 128
                             type: integer
                         required:
                         - addresses
+                        - metric
                         - prefix
                         type: object
                         x-kubernetes-validations:
@@ -674,12 +681,19 @@ spec:
                           gateway:
                             description: Gateway
                             type: string
+                          metric:
+                            default: 100
+                            description: Metric is the route priority applied to the
+                              default gateway
+                            format: int32
+                            type: integer
                           prefix:
                             description: Prefix is the network prefix to use.
                             maximum: 128
                             type: integer
                         required:
                         - addresses
+                        - metric
                         - prefix
                         type: object
                         x-kubernetes-validations:

config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclustertemplates.yaml

@@ -196,6 +196,10 @@ func getNetworkConfigDataForDevice(ctx context.Context, machineScope *scope.Mach
 	dns := machineScope.InfraCluster.ProxmoxCluster.Spec.DNSServers
 	ip := IPAddressWithPrefix(ipAddr.Spec.Address, ipAddr.Spec.Prefix)
 	gw := ipAddr.Spec.Gateway
+	metric, err := findIPAddressGatewayMetric(ctx, machineScope, ipAddr)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error converting metric annotation, kind=%s, name=%s", ipAddr.Spec.PoolRef.Kind, ipAddr.Spec.PoolRef.Name)
+	}
 
 	cloudinitNetworkConfigData := &cloudinit.NetworkConfigData{
 		MacAddress: macAddress,
@@ -205,9 +209,11 @@ func getNetworkConfigDataForDevice(ctx context.Context, machineScope *scope.Mach
 	// If it's an IPv6 address, we must set Gateway6 and IPV6Address instead
 	if strings.Contains(ip, ":") {
 		cloudinitNetworkConfigData.Gateway6 = gw
+		cloudinitNetworkConfigData.Metric6 = metric
 		cloudinitNetworkConfigData.IPV6Address = ip
 	} else {
 		cloudinitNetworkConfigData.Gateway = gw
+		cloudinitNetworkConfigData.Metric = metric
 		cloudinitNetworkConfigData.IPAddress = ip
 	}
 
@@ -241,6 +247,7 @@ func getDefaultNetworkDevice(ctx context.Context, machineScope *scope.MachineSco
 		default:
 			config.IPV6Address = conf.IPV6Address
 			config.Gateway6 = conf.Gateway6
+			config.Metric6 = conf.Metric6
 		}
 	}
 
@@ -279,17 +286,28 @@ func getCommonInterfaceConfig(ctx context.Context, machineScope *scope.MachineSc
 		if err != nil {
 			return errors.Wrapf(err, "unable to find IPAddress, device=%s", ifname)
 		}
+		metric, err := findIPAddressGatewayMetric(ctx, machineScope, ipAddr)
+		if err != nil {
+			return errors.Wrapf(err, "error converting metric annotation, kind=%s, name=%s", ipAddr.Spec.PoolRef.Kind, ipAddr.Spec.PoolRef.Name)
+		}
+
 		ciconfig.IPAddress = IPAddressWithPrefix(ipAddr.Spec.Address, ipAddr.Spec.Prefix)
 		ciconfig.Gateway = ipAddr.Spec.Gateway
+		ciconfig.Metric = metric
 	}
 	if ifconfig.IPv6PoolRef != nil && ciconfig.IPV6Address == "" {
 		var ifname = fmt.Sprintf("%s-%s", ciconfig.Name, infrav1alpha1.DefaultSuffix+"6")
 		ipAddr, err := findIPAddress(ctx, machineScope, ifname)
 		if err != nil {
 			return errors.Wrapf(err, "unable to find IPAddress, device=%s", ifname)
 		}
+		metric, err := findIPAddressGatewayMetric(ctx, machineScope, ipAddr)
+		if err != nil {
+			return errors.Wrapf(err, "error converting metric annotation, kind=%s, name=%s", ipAddr.Spec.PoolRef.Kind, ipAddr.Spec.PoolRef.Name)
+		}
 		ciconfig.IPV6Address = IPAddressWithPrefix(ipAddr.Spec.Address, ipAddr.Spec.Prefix)
 		ciconfig.Gateway6 = ipAddr.Spec.Gateway
+		ciconfig.Metric6 = metric
 	}
 
 	return nil

internal/service/vmservice/bootstrap.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net/netip"
+	"strconv"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -70,6 +71,23 @@ func findIPAddress(ctx context.Context, machineScope *scope.MachineScope, device
 	return machineScope.IPAMHelper.GetIPAddress(ctx, key)
 }
 
+func findIPAddressGatewayMetric(ctx context.Context, machineScope *scope.MachineScope, ipAddress *ipamv1.IPAddress) (*uint32, error) {
+	annotations, err := machineScope.IPAMHelper.GetIPPoolAnnotations(ctx, ipAddress)
+	if err != nil {
+		return nil, err
+	}
+	var rv *uint32
+
+	if s, exists := annotations["metric"]; exists {
+		metric, err := strconv.ParseUint(s, 0, 32)
+		if err != nil {
+			return nil, err
+		}
+		rv = ptr.To(uint32(metric))
+	}
+	return rv, nil
+}
+
 func formatIPAddressName(name, device string) string {
 	return fmt.Sprintf("%s-%s", name, device)
 }

internal/service/vmservice/ip.go

@@ -34,6 +34,9 @@ var (
 	// ErrMissingGateway returns an error if required gateway is empty.
 	ErrMissingGateway = errors.New("gateway is not set")
 
+	// ErrConflictingMetrics returns an error if a metric for a route already exists.
+	ErrConflictingMetrics = errors.New("metric already exists for default gateway")
+
 	// ErrMissingMacAddress returns an error if required mac address is empty.
 	ErrMissingMacAddress = errors.New("mac address is not set")
 

pkg/cloudinit/errors.go

@@ -88,12 +88,16 @@ const (
       routes:
        {{- if .Gateway }}
         - to: 0.0.0.0/0
-          metric: {{ if eq .Name "eth0" }}100{{ else }}200{{ end }}
+          {{- if .Metric }}
+          metric: {{ .Metric }}
+          {{- end }}
           via: {{ .Gateway }}
        {{- end }}
        {{- if .Gateway6 }}
         - to: '::/0'
-          metric: {{ if eq .Name "eth0" }}100{{ else }}200{{ end }}
+          {{- if .Metric6 }}
+          metric: {{ .Metric6 }}
+          {{- end }}
           via: '{{ .Gateway6 }}'
        {{- end }}
     {{- else }}
@@ -167,6 +171,11 @@ func (r *NetworkConfig) validate() error {
 	if len(r.data.NetworkConfigData) == 0 {
 		return ErrMissingNetworkConfigData
 	}
+	metrics := make(map[uint32]*struct {
+		ipv4 bool
+		ipv6 bool
+	})
+
 	for i, d := range r.data.NetworkConfigData {
 		// TODO: refactor this when network configuration is unified
 		if d.Type != "ethernet" {
@@ -208,6 +217,31 @@ func (r *NetworkConfig) validate() error {
 				return ErrMissingGateway
 			}
 		}
+		if d.Metric != nil {
+			if _, exists := metrics[*d.Metric]; !exists {
+				metrics[*d.Metric] = new(struct {
+					ipv4 bool
+					ipv6 bool
+				})
+			}
+			if metrics[*d.Metric].ipv4 {
+				return ErrConflictingMetrics
+			}
+			metrics[*d.Metric].ipv4 = true
+		}
+		if d.Metric6 != nil {
+			if _, exists := metrics[*d.Metric6]; !exists {
+				metrics[*d.Metric6] = new(struct {
+					ipv4 bool
+					ipv6 bool
+				})
+			}
+
+			if metrics[*d.Metric6].ipv6 {
+				return ErrConflictingMetrics
+			}
+			metrics[*d.Metric6].ipv6 = true
+		}
 	}
 	return nil
 }