feat(scheduler): allow for node overprovisioning (#47)

`ProxmoxCluster.spec.SchedulerHints.MemoryAdjustment=300` allows to (theoretically) allocate 300% of a host's memory for VMs (use with caution - enabling memory ballooning highly recommended)
`ProxmoxCluster.spec.SchedulerHints.MemoryAdjustment=95` allows to limit memory allocation to 95% of a host's memory
`ProxmoxCluster.spec.SchedulerHints.MemoryAdjustment=0` entirely disables scheduling memory constraints

Author: pborn-ionos

api/v1alpha1/proxmoxcluster_types.go

@@ -21,6 +21,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 
+	"k8s.io/utils/ptr"
 	ipamicv1 "sigs.k8s.io/cluster-api-ipam-provider-in-cluster/api/v1alpha2"
 )
 
@@ -44,6 +45,11 @@ type ProxmoxClusterSpec struct {
 	// +optional
 	AllowedNodes []string `json:"allowedNodes,omitempty"`
 
+	// SchedulerHints allows to influence the decision on where a VM will be scheduled. For example by applying a multiplicator
+	// to a node's resources, to allow for overprovisioning or to ensure a node will always have a safety buffer.
+	// +optional
+	SchedulerHints *SchedulerHints `json:"schedulerHints,omitempty"`
+
 	// IPv4Config contains information about available IPV4 address pools and the gateway.
 	// this can be combined with ipv6Config in order to enable dual stack.
 	// either IPv4Config or IPv6Config must be provided.
@@ -63,6 +69,28 @@ type ProxmoxClusterSpec struct {
 	DNSServers []string `json:"dnsServers"`
 }
 
+// SchedulerHints allows to pass the scheduler instructions to (dis)allow over- or enforce underprovisioning of resources.
+type SchedulerHints struct {
+	// MemoryAdjustment allows to adjust a node's memory by a given percentage.
+	// For example, setting it to 300 allows to allocate 300% of a host's memory for VMs,
+	// and setting it to 95 limits memory allocation to 95% of a host's memory.
+	// Setting it to 0 entirely disables scheduling memory constraints.
+	// By default 100% of a node's memory will be used for allocation.
+	// +optional
+	MemoryAdjustment *uint64 `json:"memoryAdjustment,omitempty"`
+}
+
+// GetMemoryAdjustment returns the memory adjustment percentage to use within the scheduler.
+func (sh *SchedulerHints) GetMemoryAdjustment() uint64 {
+	memoryAdjustment := uint64(100)
+
+	if sh != nil {
+		memoryAdjustment = ptr.Deref(sh.MemoryAdjustment, 100)
+	}
+
+	return memoryAdjustment
+}
+
 // ProxmoxClusterStatus defines the observed state of ProxmoxCluster.
 type ProxmoxClusterStatus struct {
 	// Ready indicates that the cluster is ready.

api/v1alpha1/zz_generated.deepcopy.go

@@ -131,6 +131,22 @@ spec:
                 x-kubernetes-validations:
                 - message: IPv6Config addresses must be provided
                   rule: self.addresses.size() > 0
+              schedulerHints:
+                description: SchedulerHints allows to influence the decision on where
+                  a VM will be scheduled. For example by applying a multiplicator
+                  to a node's resources, to allow for overprovisioning or to ensure
+                  a node will always have a safety buffer.
+                properties:
+                  memoryAdjustment:
+                    description: MemoryAdjustment allows to adjust a node's memory
+                      by a given percentage. For example, setting it to 300 allows
+                      to allocate 300% of a host's memory for VMs, and setting it
+                      to 95 limits memory allocation to 95% of a host's memory. Setting
+                      it to 0 entirely disables scheduling memory constraints. By
+                      default 100% of a node's memory will be used for allocation.
+                    format: int64
+                    type: integer
+                type: object
             required:
             - dnsServers
             type: object

config/crd/bases/infrastructure.cluster.x-k8s.io_proxmoxclusters.yaml

@@ -47,12 +47,13 @@ func (err InsufficientMemoryError) Error() string {
 func ScheduleVM(ctx context.Context, machineScope *scope.MachineScope) (string, error) {
 	client := machineScope.InfraCluster.ProxmoxClient
 	allowedNodes := machineScope.InfraCluster.ProxmoxCluster.Spec.AllowedNodes
+	schedulerHints := machineScope.InfraCluster.ProxmoxCluster.Spec.SchedulerHints
 	locations := machineScope.InfraCluster.ProxmoxCluster.Status.NodeLocations.Workers
 	if util.IsControlPlaneMachine(machineScope.Machine) {
 		locations = machineScope.InfraCluster.ProxmoxCluster.Status.NodeLocations.ControlPlane
 	}
 
-	return selectNode(ctx, client, machineScope.ProxmoxMachine, locations, allowedNodes)
+	return selectNode(ctx, client, machineScope.ProxmoxMachine, locations, allowedNodes, schedulerHints)
 }
 
 func selectNode(
@@ -61,10 +62,11 @@ func selectNode(
 	machine *infrav1.ProxmoxMachine,
 	locations []infrav1.NodeLocation,
 	allowedNodes []string,
+	schedulerHints *infrav1.SchedulerHints,
 ) (string, error) {
 	byMemory := make(sortByAvailableMemory, len(allowedNodes))
 	for i, nodeName := range allowedNodes {
-		mem, err := client.GetReservableMemoryBytes(ctx, nodeName)
+		mem, err := client.GetReservableMemoryBytes(ctx, nodeName, schedulerHints.GetMemoryAdjustment())
 		if err != nil {
 			return "", err
 		}
@@ -119,7 +121,7 @@ func selectNode(
 }
 
 type resourceClient interface {
-	GetReservableMemoryBytes(context.Context, string) (uint64, error)
+	GetReservableMemoryBytes(context.Context, string, uint64) (uint64, error)
 }
 
 type nodeInfo struct {

internal/service/scheduler/vmscheduler.go

@@ -27,7 +27,7 @@ import (
 
 type fakeResourceClient map[string]uint64
 
-func (c fakeResourceClient) GetReservableMemoryBytes(_ context.Context, nodeName string) (uint64, error) {
+func (c fakeResourceClient) GetReservableMemoryBytes(_ context.Context, nodeName string, _ uint64) (uint64, error) {
 	return c[nodeName], nil
 }
 
@@ -62,7 +62,7 @@ func TestSelectNode(t *testing.T) {
 
 			client := fakeResourceClient(availableMem)
 
-			node, err := selectNode(context.Background(), client, proxmoxMachine, locations, allowedNodes)
+			node, err := selectNode(context.Background(), client, proxmoxMachine, locations, allowedNodes, &infrav1.SchedulerHints{})
 			require.NoError(t, err)
 			require.Equal(t, expectedNode, node)
 
@@ -82,7 +82,7 @@ func TestSelectNode(t *testing.T) {
 
 		client := fakeResourceClient(availableMem)
 
-		node, err := selectNode(context.Background(), client, proxmoxMachine, locations, allowedNodes)
+		node, err := selectNode(context.Background(), client, proxmoxMachine, locations, allowedNodes, &infrav1.SchedulerHints{})
 		require.ErrorAs(t, err, &InsufficientMemoryError{})
 		require.Empty(t, node)
 

internal/service/scheduler/vmscheduler_test.go

@@ -37,7 +37,7 @@ type Client interface {
 
 	GetTask(ctx context.Context, upID string) (*proxmox.Task, error)
 
-	GetReservableMemoryBytes(ctx context.Context, nodeName string) (uint64, error)
+	GetReservableMemoryBytes(ctx context.Context, nodeName string, nodeMemoryAdjustment uint64) (uint64, error)
 
 	ResizeDisk(ctx context.Context, vm *proxmox.VirtualMachine, disk, size string) error
 

pkg/proxmox/client.go

@@ -181,13 +181,17 @@ func (c *APIClient) GetTask(ctx context.Context, upID string) (*proxmox.Task, er
 }
 
 // GetReservableMemoryBytes returns the memory that can be reserved by a new VM, in bytes.
-func (c *APIClient) GetReservableMemoryBytes(ctx context.Context, nodeName string) (uint64, error) {
+func (c *APIClient) GetReservableMemoryBytes(ctx context.Context, nodeName string, nodeMemoryAdjustment uint64) (uint64, error) {
 	node, err := c.Client.Node(ctx, nodeName)
 	if err != nil {
 		return 0, fmt.Errorf("cannot find node with name %s: %w", nodeName, err)
 	}
 
-	reservableMemory := node.Memory.Total
+	reservableMemory := uint64(float64(node.Memory.Total) / 100 * float64(nodeMemoryAdjustment))
+
+	if nodeMemoryAdjustment == 0 {
+		return node.Memory.Total, nil
+	}
 
 	vms, err := node.VirtualMachines(ctx)
 	if err != nil {

pkg/proxmox/goproxmox/api_client.go

@@ -48,13 +48,53 @@ func newJSONResponder(status int, data any) httpmock.Responder {
 
 func TestProxmoxAPIClient_GetReservableMemoryBytes(t *testing.T) {
 	tests := []struct {
-		name   string
-		maxMem uint64
-		expect uint64
+		name                 string
+		maxMem               uint64 // memory size of already provisioned guest
+		expect               uint64 // expected available memory of the host
+		nodeMemoryAdjustment uint64 // factor like 1.0 to multiply host memory with for overprovisioning
 	}{
-		{name: "under zero", maxMem: 29, expect: 1},
-		{name: "exact zero", maxMem: 30, expect: 0},
-		{name: "over zero", maxMem: 31, expect: 0},
+		{
+			name:                 "under zero - no overprovisioning",
+			maxMem:               29,
+			expect:               1,
+			nodeMemoryAdjustment: 100,
+		},
+		{
+			name:                 "exact zero - no overprovisioning",
+			maxMem:               30,
+			expect:               0,
+			nodeMemoryAdjustment: 100,
+		},
+		{
+			name:                 "over zero - no overprovisioning",
+			maxMem:               31,
+			expect:               0,
+			nodeMemoryAdjustment: 100,
+		},
+		{
+			name:                 "under zero - overprovisioning",
+			maxMem:               58,
+			expect:               2,
+			nodeMemoryAdjustment: 200,
+		},
+		{
+			name:                 "exact zero - overprovisioning",
+			maxMem:               30 * 2,
+			expect:               0,
+			nodeMemoryAdjustment: 200,
+		},
+		{
+			name:                 "over zero - overprovisioning",
+			maxMem:               31 * 2,
+			expect:               0,
+			nodeMemoryAdjustment: 200,
+		},
+		{
+			name:                 "scheduler disabled",
+			maxMem:               100,
+			expect:               30,
+			nodeMemoryAdjustment: 0,
+		},
 	}
 
 	for _, test := range tests {
@@ -104,7 +144,7 @@ func TestProxmoxAPIClient_GetReservableMemoryBytes(t *testing.T) {
 			httpmock.RegisterResponder(http.MethodGet, `=~/nodes/test/lxc`,
 				newJSONResponder(200, proxmox.Containers{}))
 
-			reservable, err := client.GetReservableMemoryBytes(context.Background(), "test")
+			reservable, err := client.GetReservableMemoryBytes(context.Background(), "test", test.nodeMemoryAdjustment)
 			require.NoError(t, err)
 			require.Equal(t, test.expect, reservable)
 		})