Merge pull request kubernetes#3532 from DataDog/azure-allocatable-tags

Azure: support allocatable resources overrides via VMSS tags

Author: k8s-ci-robot

cluster-autoscaler/cloudprovider/azure/README.md

@@ -45,6 +45,15 @@ To add the taint of `foo=bar:NoSchedule` to a node from a VMSS pool, you would a
 
 You can also use forward slashes in taints by setting them as an underscore in the tag name. For example to add the taint of `k8s.io/foo=bar:NoSchedule` to a node from a VMSS pool, you would add the following tag to the VMSS `k8s.io_cluster-autoscaler_node-template_taint_k8s.io_foo: bar:NoSchedule`
 
+#### Resources
+
+When scaling from an empty VM Scale Set (0 instances), Cluster Autoscaler will evaluate the provided presources (cpu, memory, ephemeral-storage) based on that VM Scale Set's backing instance type.
+This can be overridden (for instance, to account for system reserved resources) by specifying capacities with VMSS tags, formated as: `k8s.io_cluster-autoscaler_node-template_resources_<resource name>: <resource value>`. For instance:
+```
+k8s.io_cluster-autoscaler_node-template_resources_cpu: 3800m
+k8s.io_cluster-autoscaler_node-template_resources_memory: 11Gi
+```
+
 ## Deployment manifests
 
 Cluster autoscaler supports four Kubernetes cluster options on Azure:

cluster-autoscaler/cloudprovider/azure/azure_scale_set.go

@@ -601,6 +601,11 @@ func (scaleSet *ScaleSet) buildNodeFromTemplate(template compute.VirtualMachineS
 	node.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(vmssType.GPU, resource.DecimalSI)
 	node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(vmssType.MemoryMb*1024*1024, resource.DecimalSI)
 
+	resourcesFromTags := extractAllocatableResourcesFromScaleSet(template.Tags)
+	for resourceName, val := range resourcesFromTags {
+		node.Status.Capacity[apiv1.ResourceName(resourceName)] = *val
+	}
+
 	// TODO: set real allocatable.
 	node.Status.Allocatable = node.Status.Capacity
 
@@ -670,6 +675,25 @@ func extractTaintsFromScaleSet(tags map[string]*string) []apiv1.Taint {
 	return taints
 }
 
+func extractAllocatableResourcesFromScaleSet(tags map[string]*string) map[string]*resource.Quantity {
+	resources := make(map[string]*resource.Quantity)
+
+	for tagName, tagValue := range tags {
+		resourceName := strings.Split(tagName, nodeResourcesTagName)
+		if len(resourceName) < 2 || resourceName[1] == "" {
+			continue
+		}
+
+		quantity, err := resource.ParseQuantity(*tagValue)
+		if err != nil {
+			continue
+		}
+		resources[resourceName[1]] = &quantity
+	}
+
+	return resources
+}
+
 // TemplateNodeInfo returns a node template for this scale set.
 func (scaleSet *ScaleSet) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) {
 	template, rerr := scaleSet.getVMSSInfo()

cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient"
 	"k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/mockvmssvmclient"
@@ -533,6 +534,22 @@ func TestExtractTaintsFromScaleSet(t *testing.T) {
 	assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints))
 }
 
+func TestExtractAllocatableResourcesFromScaleSet(t *testing.T) {
+	tags := map[string]*string{
+		fmt.Sprintf("%s%s", nodeResourcesTagName, "cpu"):               to.StringPtr("100m"),
+		fmt.Sprintf("%s%s", nodeResourcesTagName, "memory"):            to.StringPtr("100M"),
+		fmt.Sprintf("%s%s", nodeResourcesTagName, "ephemeral-storage"): to.StringPtr("20G"),
+	}
+
+	labels := extractAllocatableResourcesFromScaleSet(tags)
+
+	assert.Equal(t, resource.NewMilliQuantity(100, resource.DecimalSI).String(), labels["cpu"].String())
+	expectedMemory := resource.MustParse("100M")
+	assert.Equal(t, (&expectedMemory).String(), labels["memory"].String())
+	expectedEphemeralStorage := resource.MustParse("20G")
+	assert.Equal(t, (&expectedEphemeralStorage).String(), labels["ephemeral-storage"].String())
+}
+
 func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool {
 	set := make(map[apiv1.Taint]bool)
 	for _, taint := range taints {

cluster-autoscaler/cloudprovider/azure/azure_util.go

@@ -78,8 +78,9 @@ const (
 	k8sWindowsVMAgentOrchestratorNameIndex = 2
 	k8sWindowsVMAgentPoolInfoIndex         = 3
 
-	nodeLabelTagName = "k8s.io_cluster-autoscaler_node-template_label_"
-	nodeTaintTagName = "k8s.io_cluster-autoscaler_node-template_taint_"
+	nodeLabelTagName     = "k8s.io_cluster-autoscaler_node-template_label_"
+	nodeTaintTagName     = "k8s.io_cluster-autoscaler_node-template_taint_"
+	nodeResourcesTagName = "k8s.io_cluster-autoscaler_node-template_resources_"
 )
 
 var (