Merge pull request openshift#8126 from jhixson74/master_capi_ignite_control_plane

CORS-3269: Azure ignite control plane machines

Author: openshift-merge-bot[bot]

data/data/azurestack/vnet/nsg.tf

@@ -30,4 +30,4 @@ resource "azurestack_subnet_network_security_group_association" "worker" {
 
   subnet_id                 = azurestack_subnet.worker_subnet[0].id
   network_security_group_id = azurestack_network_security_group.cluster.id
-}
+}

pkg/asset/machines/azure/azuremachines.go

@@ -179,12 +179,18 @@ func GenerateMachines(platform *azure.Platform, pool *types.MachinePool, userDat
 			},
 		},
 		Spec: capz.AzureMachineSpec{
-			VMSize:                 mpool.InstanceType,
-			Image:                  image,
-			FailureDomain:          ptr.To(mpool.Zones[0]),
-			OSDisk:                 osDisk,
-			AdditionalTags:         tags,
-			AllocatePublicIP:       true,
+			VMSize:         mpool.InstanceType,
+			Image:          image,
+			FailureDomain:  ptr.To(mpool.Zones[0]),
+			OSDisk:         osDisk,
+			AdditionalTags: tags,
+			// Do not allocate a public IP since it isn't
+			// accessible as we are using an outbound LB for the
+			// control plane. This is temporary until we have a
+			// workaround for accessing SSH	(Most likely port
+			// forwarding SSH off the LB until the bootstrap node
+			// is destroyed).
+			AllocatePublicIP:       false,
 			AdditionalCapabilities: additionalCapabilities,
 			SecurityProfile:        securityProfile,
 		},

pkg/asset/machines/azure/machines.go

@@ -298,15 +298,16 @@ func ConfigMasters(machines []machineapi.Machine, controlPlane *machinev1.Contro
 }
 
 func getNetworkInfo(platform *azure.Platform, clusterID, role string) (string, string, string, error) {
+	networkResourceGroupName := platform.NetworkResourceGroupName
 	if platform.VirtualNetwork == "" {
-		return platform.ClusterResourceGroupName(clusterID), fmt.Sprintf("%s-vnet", clusterID), fmt.Sprintf("%s-%s-subnet", clusterID, role), nil
+		networkResourceGroupName = platform.ClusterResourceGroupName(clusterID)
 	}
 
 	switch role {
 	case "worker":
-		return platform.NetworkResourceGroupName, platform.VirtualNetwork, platform.ComputeSubnet, nil
+		return networkResourceGroupName, platform.VirtualNetworkName(clusterID), platform.ComputeSubnetName(clusterID), nil
 	case "master":
-		return platform.NetworkResourceGroupName, platform.VirtualNetwork, platform.ControlPlaneSubnet, nil
+		return networkResourceGroupName, platform.VirtualNetworkName(clusterID), platform.ControlPlaneSubnetName(clusterID), nil
 	default:
 		return "", "", "", fmt.Errorf("unrecognized machine role %s", role)
 	}

pkg/asset/manifests/azure/cluster.go

@@ -3,9 +3,11 @@ package azure
 import (
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 	capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 
 	"github.com/openshift/installer/pkg/asset"
@@ -38,6 +40,29 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 	})
 
 	resourceGroup := installConfig.Config.Platform.Azure.ClusterResourceGroupName(clusterID.InfraID)
+	controlPlaneSubnet := installConfig.Config.Platform.Azure.ControlPlaneSubnetName(clusterID.InfraID)
+	networkSecurityGroup := installConfig.Config.Platform.Azure.NetworkSecurityGroupName(clusterID.InfraID)
+	computeSubnet := installConfig.Config.Platform.Azure.ComputeSubnetName(clusterID.InfraID)
+
+	securityGroup := capz.SecurityGroup{
+		Name: networkSecurityGroup,
+		SecurityGroupClass: capz.SecurityGroupClass{
+			SecurityRules: []capz.SecurityRule{
+				{
+					Name:             "apiserver_in",
+					Protocol:         capz.SecurityGroupProtocolTCP,
+					Direction:        capz.SecurityRuleDirectionInbound,
+					Priority:         101,
+					SourcePorts:      ptr.To("*"),
+					DestinationPorts: ptr.To("6443"),
+					Source:           ptr.To("*"),
+					Destination:      ptr.To("*"),
+					Action:           capz.SecurityRuleActionAllow,
+				},
+			},
+		},
+	}
+
 	azureCluster := &capz.AzureCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      clusterID.InfraID,
@@ -77,24 +102,29 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 						Type: capz.Internal,
 					},
 				},
+				ControlPlaneOutboundLB: &capz.LoadBalancerSpec{
+					FrontendIPsCount: to.Ptr(int32(1)),
+				},
 				Subnets: capz.Subnets{
 					{
 						SubnetClassSpec: capz.SubnetClassSpec{
-							Name: "control-plane-subnet",
+							Name: controlPlaneSubnet,
 							Role: capz.SubnetControlPlane,
 							CIDRBlocks: []string{
 								subnets[0].String(),
 							},
 						},
+						SecurityGroup: securityGroup,
 					},
 					{
 						SubnetClassSpec: capz.SubnetClassSpec{
-							Name: "worker-subnet",
+							Name: computeSubnet,
 							Role: capz.SubnetNode,
 							CIDRBlocks: []string{
 								subnets[1].String(),
 							},
 						},
+						SecurityGroup: securityGroup,
 					},
 				},
 			},

pkg/asset/manifests/cloudproviderconfig.go

@@ -124,7 +124,7 @@ func (cpc *CloudProviderConfig) Generate(dependencies asset.Parents) error {
 			return errors.Wrap(err, "could not get azure session")
 		}
 
-		nsg := fmt.Sprintf("%s-nsg", clusterID.InfraID)
+		nsg := installConfig.Config.Azure.NetworkSecurityGroupName(clusterID.InfraID)
 		nrg := installConfig.Config.Azure.ClusterResourceGroupName(clusterID.InfraID)
 		if installConfig.Config.Azure.NetworkResourceGroupName != "" {
 			nrg = installConfig.Config.Azure.NetworkResourceGroupName

pkg/infrastructure/azure/azure.go

@@ -168,7 +168,15 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 
 	// Create user assigned identity
 	userAssignedIdentityName := fmt.Sprintf("%s-identity", in.InfraID)
-	armmsiClientFactory, err := armmsi.NewClientFactory(subscriptionID, tokenCredential, nil)
+	armmsiClientFactory, err := armmsi.NewClientFactory(
+		subscriptionID,
+		tokenCredential,
+		&arm.ClientOptions{
+			ClientOptions: policy.ClientOptions{
+				Cloud: cloudConfiguration,
+			},
+		},
+	)
 	if err != nil {
 		return fmt.Errorf("failed to create armmsi client: %w", err)
 	}
@@ -328,7 +336,13 @@ func (p *Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput
 		return err
 	}
 
-	networkClientFactory, err := armnetwork.NewClientFactory(subscriptionID, session.TokenCreds, nil)
+	networkClientFactory, err := armnetwork.NewClientFactory(subscriptionID, session.TokenCreds,
+		&arm.ClientOptions{
+			ClientOptions: policy.ClientOptions{
+				Cloud: cloudConfiguration,
+			},
+		},
+	)
 	if err != nil {
 		return fmt.Errorf("error creating network client factory: %w", err)
 	}
@@ -393,13 +407,26 @@ func (p *Provider) PostProvision(ctx context.Context, in clusterapi.PostProvisio
 		return fmt.Errorf("error retrieving Azure session: %w", err)
 	}
 	subscriptionID := ssn.Credentials.SubscriptionID
+	cloudConfiguration := ssn.CloudConfig
 
 	if in.InstallConfig.Config.Publish == types.ExternalPublishingStrategy {
-		vmClient, err := armcompute.NewVirtualMachinesClient(subscriptionID, ssn.TokenCreds, nil)
+		vmClient, err := armcompute.NewVirtualMachinesClient(subscriptionID, ssn.TokenCreds,
+			&arm.ClientOptions{
+				ClientOptions: policy.ClientOptions{
+					Cloud: cloudConfiguration,
+				},
+			},
+		)
 		if err != nil {
 			return fmt.Errorf("error creating vm client: %w", err)
 		}
-		nicClient, err := armnetwork.NewInterfacesClient(ssn.Credentials.SubscriptionID, ssn.TokenCreds, nil)
+		nicClient, err := armnetwork.NewInterfacesClient(ssn.Credentials.SubscriptionID, ssn.TokenCreds,
+			&arm.ClientOptions{
+				ClientOptions: policy.ClientOptions{
+					Cloud: cloudConfiguration,
+				},
+			},
+		)
 		if err != nil {
 			return fmt.Errorf("error creating nic client: %w", err)
 		}

pkg/infrastructure/azure/dns.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
@@ -85,15 +87,29 @@ func createDNSEntries(ctx context.Context, in clusterapi.InfraReadyInput, extLBF
 		return fmt.Errorf("failed to create session: %w", err)
 	}
 	subscriptionID := session.Credentials.SubscriptionID
+	cloudConfiguration := session.CloudConfig
+
 	tokenCreds, err := azidentity.NewClientSecretCredential(session.Credentials.TenantID, session.Credentials.ClientID, session.Credentials.ClientSecret, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create identity: %w", err)
 	}
-	recordSetClient, err := armdns.NewRecordSetsClient(subscriptionID, tokenCreds, nil)
+	recordSetClient, err := armdns.NewRecordSetsClient(subscriptionID, tokenCreds,
+		&arm.ClientOptions{
+			ClientOptions: policy.ClientOptions{
+				Cloud: cloudConfiguration,
+			},
+		},
+	)
 	if err != nil {
 		return fmt.Errorf("failed to create public record client: %w", err)
 	}
-	privateRecordSetClient, err := armprivatedns.NewRecordSetsClient(subscriptionID, tokenCreds, nil)
+	privateRecordSetClient, err := armprivatedns.NewRecordSetsClient(subscriptionID, tokenCreds,
+		&arm.ClientOptions{
+			ClientOptions: policy.ClientOptions{
+				Cloud: cloudConfiguration,
+			},
+		},
+	)
 	if err != nil {
 		return fmt.Errorf("failed to create private record client: %w", err)
 	}

pkg/types/azure/platform.go

@@ -163,6 +163,36 @@ func (p *Platform) ClusterResourceGroupName(infraID string) string {
 	return fmt.Sprintf("%s-rg", infraID)
 }
 
+// VirtualNetworkName returns the name of the virtual network for the cluster.
+func (p *Platform) VirtualNetworkName(infraID string) string {
+	if len(p.VirtualNetwork) > 0 {
+		return p.VirtualNetwork
+	}
+	return fmt.Sprintf("%s-vnet", infraID)
+}
+
+// ControlPlaneSubnetName returns the name of the control plane subnet for the
+// cluster.
+func (p *Platform) ControlPlaneSubnetName(infraID string) string {
+	if len(p.ControlPlaneSubnet) > 0 {
+		return p.ControlPlaneSubnet
+	}
+	return fmt.Sprintf("%s-master-subnet", infraID)
+}
+
+// ComputeSubnetName returns the name of the compute subnet for the cluster.
+func (p *Platform) ComputeSubnetName(infraID string) string {
+	if len(p.ComputeSubnet) > 0 {
+		return p.ComputeSubnet
+	}
+	return fmt.Sprintf("%s-worker-subnet", infraID)
+}
+
+// NetworkSecurityGroupName returns the name of the network security group.
+func (p *Platform) NetworkSecurityGroupName(infraID string) string {
+	return fmt.Sprintf("%s-nsg", infraID)
+}
+
 // IsARO returns true if ARO-only modifications are enabled
 func (p *Platform) IsARO() bool {
 	return aro