Support cluster using network resources (VNet, LB, IP, etc.) across AAD Tenants.

Author: bowen5

staging/src/k8s.io/legacy-cloud-providers/azure/BUILD

@@ -83,6 +83,7 @@ go_library(
         "//vendor/github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage:go_default_library",
         "//vendor/github.com/Azure/azure-sdk-for-go/storage:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest:go_default_library",
+        "//vendor/github.com/Azure/go-autorest/autorest/adal:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/azure:go_default_library",
         "//vendor/github.com/Azure/go-autorest/autorest/to:go_default_library",
         "//vendor/github.com/rubiojr/go-vhd/vhd:go_default_library",

staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth.go

@@ -68,9 +68,20 @@ type AzureAuthConfig struct {
 	// ResourceManagerEndpoint is the cloud's resource manager endpoint. If set, cloud provider queries this endpoint
 	// in order to generate an autorest.Environment instance instead of using one of the pre-defined Environments.
 	ResourceManagerEndpoint string `json:"resourceManagerEndpoint,omitempty" yaml:"resourceManagerEndpoint,omitempty"`
+	// The AAD Tenant ID for the Subscription that the network resources are deployed in
+	NetworkResourceTenantID string `json:"networkResourceTenantID,omitempty" yaml:"networkResourceTenantID,omitempty"`
+	// The ID of the Azure Subscription that the network resources are deployed in
+	NetworkResourceSubscriptionID string `json:"networkResourceSubscriptionID,omitempty" yaml:"networkResourceSubscriptionID,omitempty"`
 }
 
-// GetServicePrincipalToken creates a new service principal token based on the configuration
+// GetServicePrincipalToken creates a new service principal token based on the configuration.
+//
+// By default, the cluster and its network resources are deployed in the same AAD Tenant and Subscription,
+// and all azure clients use this method to fetch Service Principal Token.
+//
+// If NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
+// than only azure clients except VM/VMSS and network resource ones use this method to fetch Token.
+// For tokens for VM/VMSS and network resource ones, please check GetMultiTenantServicePrincipalToken and GetNetworkResourceServicePrincipalToken.
 func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
 	var tenantID string
 	if strings.EqualFold(config.IdentitySystem, ADFSIdentitySystem) {
@@ -132,6 +143,75 @@ func GetServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (
 	return nil, ErrorNoAuth
 }
 
+// GetMultiTenantServicePrincipalToken is used when (and only when) NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID.
+//
+// In that scenario, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
+// and this method creates a new multi-tenant service principal token based on the configuration.
+//
+// PrimaryToken of the returned multi-tenant token is for the AAD Tenant specified by TenantID, and AuxiliaryToken of the returned multi-tenant token is for the AAD Tenant specified by NetworkResourceTenantID.
+//
+// Azure VM/VMSS clients use this multi-tenant token, in order to operate those VM/VMSS in AAD Tenant specified by TenantID, and meanwhile in their payload they are referencing network resources (e.g. Load Balancer, Network Security Group, etc.) in AAD Tenant specified by NetworkResourceTenantID.
+func GetMultiTenantServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.MultiTenantServicePrincipalToken, error) {
+	err := config.checkConfigWhenNetworkResourceInDifferentTenant()
+	if err != nil {
+		return nil, fmt.Errorf("got error(%v) in getting multi-tenant service principal token", err)
+	}
+
+	multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(
+		env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("creating the multi-tenant OAuth config: %v", err)
+	}
+
+	if len(config.AADClientSecret) > 0 {
+		klog.V(2).Infoln("azure: using client_id+client_secret to retrieve multi-tenant access token")
+		return adal.NewMultiTenantServicePrincipalToken(
+			multiTenantOAuthConfig,
+			config.AADClientID,
+			config.AADClientSecret,
+			env.ServiceManagementEndpoint)
+	}
+
+	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
+		return nil, fmt.Errorf("AAD Application client certificate authentication is not supported in getting multi-tenant service principal token")
+	}
+
+	return nil, ErrorNoAuth
+}
+
+// GetNetworkResourceServicePrincipalToken is used when (and only when) NetworkResourceTenantID and NetworkResourceSubscriptionID are specified to have different values than TenantID and SubscriptionID.
+//
+// In that scenario, network resources are deployed in different AAD Tenant and Subscription than those for the cluster,
+// and this method creates a new service principal token for network resources tenant based on the configuration.
+//
+// Azure network resource (Load Balancer, Public IP, Route Table, Network Security Group and their sub level resources) clients use this multi-tenant token, in order to operate resources in AAD Tenant specified by NetworkResourceTenantID.
+func GetNetworkResourceServicePrincipalToken(config *AzureAuthConfig, env *azure.Environment) (*adal.ServicePrincipalToken, error) {
+	err := config.checkConfigWhenNetworkResourceInDifferentTenant()
+	if err != nil {
+		return nil, fmt.Errorf("got error(%v) in getting network resources service principal token", err)
+	}
+
+	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating the OAuth config for network resources tenant: %v", err)
+	}
+
+	if len(config.AADClientSecret) > 0 {
+		klog.V(2).Infoln("azure: using client_id+client_secret to retrieve access token for network resources tenant")
+		return adal.NewServicePrincipalToken(
+			*oauthConfig,
+			config.AADClientID,
+			config.AADClientSecret,
+			env.ServiceManagementEndpoint)
+	}
+
+	if len(config.AADClientCertPath) > 0 && len(config.AADClientCertPassword) > 0 {
+		return nil, fmt.Errorf("AAD Application client certificate authentication is not supported in getting network resources service principal token")
+	}
+
+	return nil, ErrorNoAuth
+}
+
 // ParseAzureEnvironment returns the azure environment.
 // If 'resourceManagerEndpoint' is set, the environment is computed by quering the cloud's resource manager endpoint.
 // Otherwise, a pre-defined Environment is looked up by name.
@@ -155,6 +235,16 @@ func ParseAzureEnvironment(cloudName, resourceManagerEndpoint, identitySystem st
 	return &env, err
 }
 
+// UsesNetworkResourceInDifferentTenant determines whether the AzureAuthConfig indicates to use network resources in different AAD Tenant and Subscription than those for the cluster
+// Return true only when both NetworkResourceTenantID and NetworkResourceSubscriptionID are specified
+// and they are not equals to TenantID and SubscriptionID
+func (config *AzureAuthConfig) UsesNetworkResourceInDifferentTenant() bool {
+	return len(config.NetworkResourceTenantID) > 0 &&
+		len(config.NetworkResourceSubscriptionID) > 0 &&
+		!strings.EqualFold(config.NetworkResourceTenantID, config.TenantID) &&
+		!strings.EqualFold(config.NetworkResourceSubscriptionID, config.SubscriptionID)
+}
+
 // decodePkcs12 decodes a PKCS#12 client certificate by extracting the public certificate and
 // the private RSA key
 func decodePkcs12(pkcs []byte, password string) (*x509.Certificate, *rsa.PrivateKey, error) {
@@ -181,3 +271,20 @@ func azureStackOverrides(env *azure.Environment, resourceManagerEndpoint, identi
 		env.ActiveDirectoryEndpoint = strings.TrimSuffix(env.ActiveDirectoryEndpoint, "adfs")
 	}
 }
+
+// checkConfigWhenNetworkResourceInDifferentTenant checks configuration for the scenario of using network resource in different tenant
+func (config *AzureAuthConfig) checkConfigWhenNetworkResourceInDifferentTenant() error {
+	if !config.UsesNetworkResourceInDifferentTenant() {
+		return fmt.Errorf("NetworkResourceTenantID and NetworkResourceSubscriptionID must be configured")
+	}
+
+	if strings.EqualFold(config.IdentitySystem, ADFSIdentitySystem) {
+		return fmt.Errorf("ADFS identity system is not supported")
+	}
+
+	if config.UseManagedIdentityExtension {
+		return fmt.Errorf("managed identity is not supported")
+	}
+
+	return nil
+}

staging/src/k8s.io/legacy-cloud-providers/azure/auth/azure_auth_test.go

@@ -24,6 +24,32 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+var (
+	CrossTenantNetworkResourceNegativeConfig = []*AzureAuthConfig{
+		{
+			TenantID:        "TenantID",
+			AADClientID:     "AADClientID",
+			AADClientSecret: "AADClientSecret",
+		},
+		{
+			TenantID:                      "TenantID",
+			AADClientID:                   "AADClientID",
+			AADClientSecret:               "AADClientSecret",
+			NetworkResourceTenantID:       "NetworkResourceTenantID",
+			NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+			IdentitySystem:                ADFSIdentitySystem,
+		},
+		{
+			TenantID:                      "TenantID",
+			AADClientID:                   "AADClientID",
+			AADClientSecret:               "AADClientSecret",
+			NetworkResourceTenantID:       "NetworkResourceTenantID",
+			NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+			UseManagedIdentityExtension:   true,
+		},
+	}
+)
+
 func TestGetServicePrincipalTokenFromMSIWithUserAssignedID(t *testing.T) {
 	configs := []*AzureAuthConfig{
 		{
@@ -106,6 +132,66 @@ func TestGetServicePrincipalToken(t *testing.T) {
 	assert.Equal(t, token, spt)
 }
 
+func TestGetMultiTenantServicePrincipalToken(t *testing.T) {
+	config := &AzureAuthConfig{
+		TenantID:                      "TenantID",
+		AADClientID:                   "AADClientID",
+		AADClientSecret:               "AADClientSecret",
+		NetworkResourceTenantID:       "NetworkResourceTenantID",
+		NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+	}
+	env := &azure.PublicCloud
+
+	multiTenantToken, err := GetMultiTenantServicePrincipalToken(config, env)
+	assert.NoError(t, err)
+
+	multiTenantOAuthConfig, err := adal.NewMultiTenantOAuthConfig(env.ActiveDirectoryEndpoint, config.TenantID, []string{config.NetworkResourceTenantID}, adal.OAuthOptions{})
+	assert.NoError(t, err)
+
+	spt, err := adal.NewMultiTenantServicePrincipalToken(multiTenantOAuthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint)
+	assert.NoError(t, err)
+
+	assert.Equal(t, multiTenantToken, spt)
+}
+
+func TestGetMultiTenantServicePrincipalTokenNegative(t *testing.T) {
+	env := &azure.PublicCloud
+	for _, config := range CrossTenantNetworkResourceNegativeConfig {
+		_, err := GetMultiTenantServicePrincipalToken(config, env)
+		assert.Error(t, err)
+	}
+}
+
+func TestGetNetworkResourceServicePrincipalToken(t *testing.T) {
+	config := &AzureAuthConfig{
+		TenantID:                      "TenantID",
+		AADClientID:                   "AADClientID",
+		AADClientSecret:               "AADClientSecret",
+		NetworkResourceTenantID:       "NetworkResourceTenantID",
+		NetworkResourceSubscriptionID: "NetworkResourceSubscriptionID",
+	}
+	env := &azure.PublicCloud
+
+	token, err := GetNetworkResourceServicePrincipalToken(config, env)
+	assert.NoError(t, err)
+
+	oauthConfig, err := adal.NewOAuthConfigWithAPIVersion(env.ActiveDirectoryEndpoint, config.NetworkResourceTenantID, nil)
+	assert.NoError(t, err)
+
+	spt, err := adal.NewServicePrincipalToken(*oauthConfig, config.AADClientID, config.AADClientSecret, env.ServiceManagementEndpoint)
+	assert.NoError(t, err)
+
+	assert.Equal(t, token, spt)
+}
+
+func TestGetNetworkResourceServicePrincipalTokenNegative(t *testing.T) {
+	env := &azure.PublicCloud
+	for _, config := range CrossTenantNetworkResourceNegativeConfig {
+		_, err := GetNetworkResourceServicePrincipalToken(config, env)
+		assert.Error(t, err)
+	}
+}
+
 func TestParseAzureEngironment(t *testing.T) {
 	cases := []struct {
 		cloudName               string