diff --git a/cluster-autoscaler/cloudprovider/azure/azure_cache.go b/cluster-autoscaler/cloudprovider/azure/azure_cache.go index cc3ded38e33f..08c7cfea28ed 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_cache.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_cache.go @@ -107,11 +107,17 @@ type azureCache struct { } func newAzureCache(client *azClient, cacheTTL time.Duration, config Config) (*azureCache, error) { + nodeResourceGroup := config.ResourceGroup + // Hosted (on-behalf-of) system pool node resources are in the AKS internal resource group within AME tenants, + // which differs from the MC_* resource group found in the customer subscription. + if config.HostedResourceGroup != "" { + nodeResourceGroup = config.HostedResourceGroup + } cache := &azureCache{ interrupt: make(chan struct{}), azClient: client, refreshInterval: cacheTTL, - resourceGroup: config.ResourceGroup, + resourceGroup: nodeResourceGroup, clusterResourceGroup: config.ClusterResourceGroup, clusterName: config.ClusterName, enableVMsAgentPool: config.EnableVMsAgentPool, diff --git a/cluster-autoscaler/cloudprovider/azure/azure_config.go b/cluster-autoscaler/cloudprovider/azure/azure_config.go index df11d55c85c9..7af2f9710454 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_config.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_config.go @@ -65,6 +65,15 @@ type Config struct { // It can override the default public ARM endpoint for VMs pool scale operations. ARMBaseURLForAPClient string `json:"armBaseURLForAPClient" yaml:"armBaseURLForAPClient"` + // Hosted (on-behalf-of) system pool configuration for automatic cluster. + // HostedSubscriptionID is the subscription ID of the hosted resources under AKS internal tenant. + HostedSubscriptionID string `json:"hostedSubscriptionID" yaml:"hostedSubscriptionID"` + // HostedResourceGroup is the resource group of the hosted resources under AKS internal tenant. + HostedResourceGroup string `json:"hostedResourceGroup" yaml:"hostedResourceGroup"` + // HostedResourceProxyURL is the URL to use for retrieving hosted resources under AKS internal tenant. + // It can override the default public ARM endpoint for operations like VM/SKU GET. + HostedResourceProxyURL string `json:"hostedResourceProxyURL" yaml:"hostedResourceProxyURL"` + // AuthMethod determines how to authorize requests for the Azure // cloud. Valid options are "principal" (= the traditional // service principle approach) and "cli" (= load az command line @@ -223,6 +232,15 @@ func BuildAzureConfig(configReader io.Reader) (*Config, error) { if _, err = assignFromEnvIfExists(&cfg.SubscriptionID, "ARM_SUBSCRIPTION_ID"); err != nil { return nil, err } + if _, err = assignFromEnvIfExists(&cfg.HostedResourceProxyURL, "HOSTED_RESOURCE_PROXY_URL"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.HostedSubscriptionID, "HOSTED_SUBSCRIPTION_ID"); err != nil { + return nil, err + } + if _, err = assignFromEnvIfExists(&cfg.HostedResourceGroup, "HOSTED_RESOURCE_GROUP"); err != nil { + return nil, err + } if _, err = assignBoolFromEnvIfExists(&cfg.UseManagedIdentityExtension, "ARM_USE_MANAGED_IDENTITY_EXTENSION"); err != nil { return nil, err } @@ -380,6 +398,17 @@ func (cfg *Config) getAzureClientConfig(authorizer autorest.Authorizer, env *azu } } + // A proxy service is required to access resources for the Hosted (on-behalf-of) system pool within automatic clusters. + if cfg.HostedResourceProxyURL != "" { + azClientConfig.ResourceManagerEndpoint = cfg.HostedResourceProxyURL + } + + // Hosted (on-behalf-of) system pool resources are hosted under AKS internal tenant and subscription. + // it is different from the customer subscription where the cluster is created. + if cfg.HostedSubscriptionID != "" { + azClientConfig.SubscriptionID = cfg.HostedSubscriptionID + } + if cfg.HasExtendedLocation() { azClientConfig.ExtendedLocation = &azclients.ExtendedLocation{ Name: cfg.ExtendedLocationName, diff --git a/cluster-autoscaler/cloudprovider/azure/azure_vms_pool.go b/cluster-autoscaler/cloudprovider/azure/azure_vms_pool.go index 7b369a6822cb..577722e0b2b6 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_vms_pool.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_vms_pool.go @@ -154,9 +154,16 @@ func (vmPool *VMPool) IncreaseSize(delta int) error { if len(versionedAP.Properties.VirtualMachinesProfile.Scale.Manual) > 0 { requestBody = buildRequestBodyForScaleUp(versionedAP, count, vmPool.sku) - } else { // AKS-managed CAS will use custom header for setting the target count + } + // hosted CAS will be using Autoscale scale profile + // HostedSystem will be using manual scale profile + // Both of them need to set the Target-Count and SKU headers + if len(versionedAP.Properties.VirtualMachinesProfile.Scale.Autoscale) > 0 || + (versionedAP.Properties.Mode != nil && + strings.EqualFold(string(*versionedAP.Properties.Mode), "HostedSystem")) { header := make(http.Header) header.Set("Target-Count", fmt.Sprintf("%d", count)) + header.Set("SKU", fmt.Sprintf("%s", vmPool.sku)) updateCtx = policy.WithHTTPHeader(updateCtx, header) }