Add support for hosted-on-behalf-of systempool autoscaling

Author: wenxuan0923

cluster-autoscaler/cloudprovider/azure/azure_cache.go

@@ -107,11 +107,15 @@ type azureCache struct {
 }
 
 func newAzureCache(client *azClient, cacheTTL time.Duration, config Config) (*azureCache, error) {
+	nodeResourceGroup := config.ResourceGroup
+	if config.ManagedResourceGroup != "" {
+		nodeResourceGroup = config.ManagedResourceGroup
+	}
 	cache := &azureCache{
 		interrupt:            make(chan struct{}),
 		azClient:             client,
 		refreshInterval:      cacheTTL,
-		resourceGroup:        config.ResourceGroup,
+		resourceGroup:        nodeResourceGroup,
 		clusterResourceGroup: config.ClusterResourceGroup,
 		clusterName:          config.ClusterName,
 		enableVMsAgentPool:   config.EnableVMsAgentPool,

cluster-autoscaler/cloudprovider/azure/azure_config.go

@@ -65,6 +65,15 @@ type Config struct {
 	// It can override the default public ARM endpoint for VMs pool scale operations.
 	ARMBaseURLForAPClient string `json:"armBaseURLForAPClient" yaml:"armBaseURLForAPClient"`
 
+	// Managed system pool configuration for automatic cluster.
+	// ManagedSubscriptionID is the subscription ID of the managed resources under AKS internal tenant.
+	ManagedSubscriptionID string `json:"managedSubscriptionID" yaml:"managedSubscriptionID"`
+	// ManagedResourceGroup is the resource group of the managed resources under AKS internal tenant.
+	ManagedResourceGroup string `json:"managedResourceGroup" yaml:"managedResourceGroup"`
+	// ManagedResourceProxyURL is the URL to use for retrieving managed resources under AKS internal tenant.
+	// It can override the default public ARM endpoint for operations like VM/SKU GET.
+	ManagedResourceProxyURL string `json:"managedResourceProxyURL" yaml:"managedResourceProxyURL"`
+
 	// AuthMethod determines how to authorize requests for the Azure
 	// cloud. Valid options are "principal" (= the traditional
 	// service principle approach) and "cli" (= load az command line
@@ -223,6 +232,15 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) {
 	if _, err = assignFromEnvIfExists(&cfg.SubscriptionID, "ARM_SUBSCRIPTION_ID"); err != nil {
 		return nil, err
 	}
+	if _, err = assignFromEnvIfExists(&cfg.ManagedResourceProxyURL, "MANAGED_RESOURCE_PROXY_URL"); err != nil {
+		return nil, err
+	}
+	if _, err = assignFromEnvIfExists(&cfg.ManagedSubscriptionID, "MANAGED_SUBSCRIPTION_ID"); err != nil {
+		return nil, err
+	}
+	if _, err = assignFromEnvIfExists(&cfg.ManagedResourceGroup, "MANAGED_RESOURCE_GROUP"); err != nil {
+		return nil, err
+	}
 	if _, err = assignBoolFromEnvIfExists(&cfg.UseManagedIdentityExtension, "ARM_USE_MANAGED_IDENTITY_EXTENSION"); err != nil {
 		return nil, err
 	}
@@ -387,6 +405,17 @@ func (cfg *Config) getAzureClientConfig(authorizer autorest.Authorizer, env *azu
 		}
 	}
 
+	// A proxy service is required to access resources for the managed system pool within automatic clusters.
+	if cfg.ManagedResourceProxyURL != "" {
+		azClientConfig.ResourceManagerEndpoint = cfg.ManagedResourceProxyURL
+	}
+
+	// Managed system pool resources are hosted under AKS internal tenant and subscription.
+	// it is different from the customer subscription where the cluster is created.
+	if cfg.ManagedSubscriptionID != "" {
+		azClientConfig.SubscriptionID = cfg.ManagedSubscriptionID
+	}
+
 	return azClientConfig
 }
 

cluster-autoscaler/cloudprovider/azure/azure_vms_pool.go

@@ -154,11 +154,11 @@ func (vmPool *VMPool) IncreaseSize(delta int) error {
 	if len(versionedAP.Properties.VirtualMachinesProfile.Scale.Manual) > 0 {
 		requestBody = buildRequestBodyForScaleUp(versionedAP, count, vmPool.sku)
 
-	} else { // AKS-managed CAS will use custom header for setting the target count
-		header := make(http.Header)
-		header.Set("Target-Count", fmt.Sprintf("%d", count))
-		updateCtx = policy.WithHTTPHeader(updateCtx, header)
 	}
+	header := make(http.Header)
+	header.Set("Target-Count", fmt.Sprintf("%d", count))
+	header.Set("SKU", fmt.Sprintf("%s", vmPool.sku))
+	updateCtx = policy.WithHTTPHeader(updateCtx, header)
 
 	defer vmPool.manager.invalidateCache()
 	poller, err := vmPool.manager.azClient.agentPoolClient.BeginCreateOrUpdate(