Azure: support allocatable resources overrides via VMSS tags

This allows to specify effective nodes resources capacity using Scale
Sets tags, preventing wrong CA decisions and infinite upscale when pods
requests are within instance type capacity but over k8s nodes allocatable
(which might comprise system and kubelet's reservations), and when using
node-infos from instances templates (ie. scaling from 0).

This is similar to what AWS (with launch configuration tags) and
GCP (with instances templates metadata) cloud providers offers,
ensuring the tags format is similar to AWS' for consistency.

See also:
https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/proposals/min_at_zero_gcp.md

Author: bpineau

cluster-autoscaler/cloudprovider/azure/README.md

@@ -45,6 +45,15 @@ To add the taint of `foo=bar:NoSchedule` to a node from a VMSS pool, you would a
 
 You can also use forward slashes in taints by setting them as an underscore in the tag name. For example to add the taint of `k8s.io/foo=bar:NoSchedule` to a node from a VMSS pool, you would add the following tag to the VMSS `k8s.io_cluster-autoscaler_node-template_taint_k8s.io_foo: bar:NoSchedule`
 
+#### Resources
+
+When scaling from an empty VM Scale Set (0 instances), Cluster Autoscaler will evaluate the provided presources (cpu, memory, ephemeral-storage) based on that VM Scale Set's backing instance type.
+This can be overridden (for instance, to account for system reserved resources) by specifying capacities with VMSS tags, formated as: `k8s.io_cluster-autoscaler_node-template_resources_<resource name>: <resource value>`. For instance:
+```
+k8s.io_cluster-autoscaler_node-template_resources_cpu: 3800m
+k8s.io_cluster-autoscaler_node-template_resources_memory: 11Gi
+```
+
 ## Deployment manifests
 
 Cluster autoscaler supports four Kubernetes cluster options on Azure:

cluster-autoscaler/cloudprovider/azure/azure_scale_set.go

@@ -601,6 +601,11 @@ func (scaleSet *ScaleSet) buildNodeFromTemplate(template compute.VirtualMachineS
 	node.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(vmssType.GPU, resource.DecimalSI)
 	node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(vmssType.MemoryMb*1024*1024, resource.DecimalSI)
 
+	resourcesFromTags := extractAllocatableResourcesFromScaleSet(template.Tags)
+	for resourceName, val := range resourcesFromTags {
+		node.Status.Capacity[apiv1.ResourceName(resourceName)] = *val
+	}
+
 	// TODO: set real allocatable.
 	node.Status.Allocatable = node.Status.Capacity
 
@@ -670,6 +675,25 @@ func extractTaintsFromScaleSet(tags map[string]*string) []apiv1.Taint {
 	return taints
 }
 
+func extractAllocatableResourcesFromScaleSet(tags map[string]*string) map[string]*resource.Quantity {
+	resources := make(map[string]*resource.Quantity)
+
+	for tagName, tagValue := range tags {
+		resourceName := strings.Split(tagName, nodeResourcesTagName)
+		if len(resourceName) < 2 || resourceName[1] == "" {
+			continue
+		}
+
+		quantity, err := resource.ParseQuantity(*tagValue)
+		if err != nil {
+			continue
+		}
+		resources[resourceName[1]] = &quantity
+	}
+
+	return resources
+}
+
 // TemplateNodeInfo returns a node template for this scale set.
 func (scaleSet *ScaleSet) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) {
 	template, rerr := scaleSet.getVMSSInfo()

cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/mockvmssvmclient"
@@ -533,6 +534,22 @@ func TestExtractTaintsFromScaleSet(t *testing.T) {
 	assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints))
 }
 
+func TestExtractAllocatableResourcesFromScaleSet(t *testing.T) {
+	tags := map[string]*string{
+		fmt.Sprintf("%s%s", nodeResourcesTagName, "cpu"):               to.StringPtr("100m"),
+		fmt.Sprintf("%s%s", nodeResourcesTagName, "memory"):            to.StringPtr("100M"),
+		fmt.Sprintf("%s%s", nodeResourcesTagName, "ephemeral-storage"): to.StringPtr("20G"),
+	}
+
+	labels := extractAllocatableResourcesFromScaleSet(tags)
+
+	assert.Equal(t, resource.NewMilliQuantity(100, resource.DecimalSI).String(), labels["cpu"].String())
+	expectedMemory := resource.MustParse("100M")
+	assert.Equal(t, (&expectedMemory).String(), labels["memory"].String())
+	expectedEphemeralStorage := resource.MustParse("20G")
+	assert.Equal(t, (&expectedEphemeralStorage).String(), labels["ephemeral-storage"].String())
+}
+
 func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool {
 	set := make(map[apiv1.Taint]bool)
 	for _, taint := range taints {

cluster-autoscaler/cloudprovider/azure/azure_util.go

@@ -78,8 +78,9 @@ const (
 	k8sWindowsVMAgentOrchestratorNameIndex = 2
 	k8sWindowsVMAgentPoolInfoIndex         = 3
 
-	nodeLabelTagName = "k8s.io_cluster-autoscaler_node-template_label_"
-	nodeTaintTagName = "k8s.io_cluster-autoscaler_node-template_taint_"
+	nodeLabelTagName     = "k8s.io_cluster-autoscaler_node-template_label_"
+	nodeTaintTagName     = "k8s.io_cluster-autoscaler_node-template_taint_"
+	nodeResourcesTagName = "k8s.io_cluster-autoscaler_node-template_resources_"
 )
 
 var (