fix AzureManagedCluster deployments when the resource group is not present

Author: devigned

Tiltfile

@@ -14,6 +14,8 @@ settings = {
     "kind_cluster_name": "capz",
     "capi_version": "v0.3.7",
     "cert_manager_version": "v0.11.0",
+    "kubernetes_version": "v1.18.6",
+    "aks_kubernetes_version": "v1.17.7"
 }
 
 keys = ["AZURE_SUBSCRIPTION_ID_B64", "AZURE_TENANT_ID_B64", "AZURE_CLIENT_SECRET_B64", "AZURE_CLIENT_ID_B64", "AZURE_ENVIRONMENT"]
@@ -152,8 +154,7 @@ COPY manager .
 def capz():
     # Apply the kustomized yaml for this provider
     substitutions = settings.get("kustomize_substitutions", {})
-    for substitution in substitutions:
-        os.putenv(substitution, substitutions[substitution])
+    os.environ.update(substitutions)
     yaml = str(kustomizesub("./config"))
 
 
@@ -266,13 +267,17 @@ def deploy_worker_templates(flavor, substitutions):
         "AZURE_VNET_NAME": flavor + "-template-vnet",
         "AZURE_RESOURCE_GROUP": flavor + "-template-rg",
         "CONTROL_PLANE_MACHINE_COUNT": "1",
-        "KUBERNETES_VERSION": "v1.18.6",
+        "KUBERNETES_VERSION": settings.get("kubernetes_version"),
         "AZURE_CONTROL_PLANE_MACHINE_TYPE": "Standard_D2s_v3",
         "WORKER_MACHINE_COUNT": "2",
         "AZURE_NODE_MACHINE_TYPE": "Standard_D2s_v3",
         "AZURE_JSON_B64": base64_encode(azure_json(flavor, substitutions)),
     }
 
+    if flavor == "aks":
+        # AKS version support is usually a bit behind CAPI version, so use an older version
+        substitutions["KUBERNETES_VERSION"] = settings.get("aks_kubernetes_version")
+
     for substitution in substitutions:
         value = substitutions[substitution]
         yaml = yaml.replace("${" + substitution + "}", value)

cloud/scope/managedcontrolplane.go

@@ -18,15 +18,18 @@ package scope
 
 import (
 	"context"
+
 	"github.com/Azure/go-autorest/autorest"
 	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/klog/klogr"
-	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha3"
 
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
+	infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3"
+
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -94,6 +97,33 @@ type ManagedControlPlaneScope struct {
 	PatchTarget      runtime.Object
 }
 
+func (s *ManagedControlPlaneScope) ResourceGroup() string {
+	if s.ControlPlane == nil {
+		return ""
+	}
+	return s.ControlPlane.Spec.ResourceGroup
+}
+
+func (s *ManagedControlPlaneScope) ClusterName() string {
+	return s.Cluster.Name
+}
+
+func (s *ManagedControlPlaneScope) Location() string {
+	if s.ControlPlane == nil {
+		return ""
+	}
+	return s.ControlPlane.Spec.Location
+}
+
+// AdditionalTags returns AdditionalTags from the ControlPlane spec.
+func (s *ManagedControlPlaneScope) AdditionalTags() infrav1.Tags {
+	tags := make(infrav1.Tags)
+	if s.ControlPlane.Spec.AdditionalTags != nil {
+		tags = s.ControlPlane.Spec.AdditionalTags.DeepCopy()
+	}
+	return tags
+}
+
 // SubscriptionID returns the Azure client Subscription ID.
 func (s *ManagedControlPlaneScope) SubscriptionID() string {
 	return s.AzureClients.SubscriptionID

cloud/services/groups/mock_groups/groups_mock.go

@@ -18,6 +18,7 @@ package groups
 
 import (
 	"github.com/go-logr/logr"
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
 	azure "sigs.k8s.io/cluster-api-provider-azure/cloud"
 )
 
@@ -30,7 +31,11 @@ type Service struct {
 // GroupScope defines the scope interface for a group service.
 type GroupScope interface {
 	logr.Logger
-	azure.ClusterDescriber
+	azure.Authorizer
+	ResourceGroup() string
+	ClusterName() string
+	Location() string
+	AdditionalTags() infrav1.Tags
 }
 
 // NewService creates a new service.

cloud/services/groups/service.go

@@ -98,7 +98,7 @@ spec:
                 type: string
               sshPublicKey:
                 description: SSHPublicKey is a string literal containing an ssh public
-                  key.
+                  key base64 encoded.
                 type: string
               subscriptionID:
                 description: SubscriotionID is the GUID of the Azure subscription

config/crd/bases/exp.infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml

@@ -20,6 +20,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
+
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
 )
 
 // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane
@@ -44,7 +46,7 @@ type AzureManagedControlPlaneSpec struct {
 	// AdditionalTags is an optional set of tags to add to Azure resources managed by the Azure provider, in addition to the
 	// ones added by default.
 	// +optional
-	AdditionalTags map[string]string `json:"additionalTags,omitempty"`
+	AdditionalTags infrav1.Tags `json:"additionalTags,omitempty"`
 
 	// NetworkPlugin used for building Kubernetes network. Possible values include: 'Azure', 'Kubenet'. Defaults to Azure.
 	// +kubebuilder:validation:Enum=Azure;Kubenet
@@ -54,7 +56,7 @@ type AzureManagedControlPlaneSpec struct {
 	// +kubebuilder:validation:Enum=Calico;Azure
 	NetworkPolicy *string `json:"networkPolicy,omitempty"`
 
-	// SSHPublicKey is a string literal containing an ssh public key.
+	// SSHPublicKey is a string literal containing an ssh public key base64 encoded.
 	SSHPublicKey string `json:"sshPublicKey"`
 
 	// DefaultPoolRef is the specification for the default pool, without which an AKS cluster cannot be created.

exp/api/v1alpha3/azuremanagedcontrolplane_types.go

@@ -168,6 +168,13 @@ func (r *AzureManagedMachinePoolReconciler) Reconcile(req ctrl.Request) (_ ctrl.
 		return reconcile.Result{}, errors.Errorf("failed to create scope: %+v", err)
 	}
 
+	// Always patch when exiting so we can persist changes to finalizers and status
+	defer func() {
+		if err := mcpScope.PatchObject(ctx); err != nil && reterr == nil {
+			reterr = err
+		}
+	}()
+
 	// Handle deleted clusters
 	if !infraPool.DeletionTimestamp.IsZero() {
 		return r.reconcileDelete(ctx, mcpScope)
@@ -188,6 +195,12 @@ func (r *AzureManagedMachinePoolReconciler) reconcileNormal(ctx context.Context,
 	}
 
 	if err := newAzureManagedMachinePoolReconciler(scope).Reconcile(ctx, scope); err != nil {
+		if IsAgentPoolVMSSNotFoundError(err) {
+			// if the underlying VMSS is not yet created, requeue for 30s in the future
+			return reconcile.Result{
+				RequeueAfter: 30 * time.Second,
+			}, nil
+		}
 		return reconcile.Result{}, errors.Wrapf(err, "error creating AzureManagedMachinePool %s/%s", scope.InfraMachinePool.Namespace, scope.InfraMachinePool.Name)
 	}
 

exp/api/v1alpha3/zz_generated.deepcopy.go

@@ -30,17 +30,47 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
-// azureManagedMachinePoolReconciler are list of services required by cluster controller
-type azureManagedMachinePoolReconciler struct {
-	kubeclient    client.Client
-	agentPoolsSvc azure.OldService
-	scaleSetsSvc  NodeLister
+type (
+	// azureManagedMachinePoolReconciler are list of services required by cluster controller
+	azureManagedMachinePoolReconciler struct {
+		kubeclient    client.Client
+		agentPoolsSvc azure.OldService
+		scaleSetsSvc  NodeLister
+	}
+
+	// AgentPoolVMSSNotFoundError represents a reconcile error when the VMSS for an agent pool can't be found
+	AgentPoolVMSSNotFoundError struct {
+		NodeResourceGroup string
+		PoolName          string
+	}
+
+	// NodeLister is a service interface for returning generic lists.
+	NodeLister interface {
+		ListInstances(context.Context, string, string) ([]compute.VirtualMachineScaleSetVM, error)
+		List(context.Context, string) ([]compute.VirtualMachineScaleSet, error)
+	}
+)
+
+var (
+	notFoundErr = new(AgentPoolVMSSNotFoundError)
+)
+
+// NewAgentPoolVMSSNotFoundError creates a new AgentPoolVMSSNotFoundError
+func NewAgentPoolVMSSNotFoundError(nodeResourceGroup, poolName string) *AgentPoolVMSSNotFoundError {
+	return &AgentPoolVMSSNotFoundError{
+		NodeResourceGroup: nodeResourceGroup,
+		PoolName:          poolName,
+	}
 }
 
-// NodeLister is a service interface for returning generic lists.
-type NodeLister interface {
-	ListInstances(context.Context, string, string) ([]compute.VirtualMachineScaleSetVM, error)
-	List(context.Context, string) ([]compute.VirtualMachineScaleSet, error)
+func (a *AgentPoolVMSSNotFoundError) Error() string {
+	msgFmt := "failed to find vm scale set in resource group %s matching pool named %s"
+	return fmt.Sprintf(msgFmt, a.NodeResourceGroup, a.PoolName)
+}
+
+func (a *AgentPoolVMSSNotFoundError) Is(target error) bool {
+	_, ok := target.(*AgentPoolVMSSNotFoundError)
+	return ok
 }
 
 // newAzureManagedMachinePoolReconciler populates all the services based on input scope
@@ -62,23 +92,24 @@ func (r *azureManagedMachinePoolReconciler) Reconcile(ctx context.Context, scope
 		normalizedVersion = &v
 	}
 
+	replicas := int32(1)
+	if scope.MachinePool.Spec.Replicas != nil {
+		replicas = *scope.MachinePool.Spec.Replicas
+	}
+
 	agentPoolSpec := &agentpools.Spec{
 		Name:          scope.InfraMachinePool.Name,
 		ResourceGroup: scope.ControlPlane.Spec.ResourceGroup,
 		Cluster:       scope.ControlPlane.Name,
 		SKU:           scope.InfraMachinePool.Spec.SKU,
-		Replicas:      1,
+		Replicas:      replicas,
 		Version:       normalizedVersion,
 	}
 
 	if scope.InfraMachinePool.Spec.OSDiskSizeGB != nil {
 		agentPoolSpec.OSDiskSizeGB = *scope.InfraMachinePool.Spec.OSDiskSizeGB
 	}
 
-	if scope.MachinePool.Spec.Replicas != nil {
-		agentPoolSpec.Replicas = *scope.MachinePool.Spec.Replicas
-	}
-
 	if err := r.agentPoolsSvc.Reconcile(ctx, agentPoolSpec); err != nil {
 		return errors.Wrapf(err, "failed to reconcile machine pool %s", scope.InfraMachinePool.Name)
 	}
@@ -99,7 +130,7 @@ func (r *azureManagedMachinePoolReconciler) Reconcile(ctx context.Context, scope
 	}
 
 	if match == nil {
-		return errors.New("failed to find vm scale set matching pool")
+		return NewAgentPoolVMSSNotFoundError(nodeResourceGroup, scope.InfraMachinePool.Name)
 	}
 
 	instances, err := r.scaleSetsSvc.ListInstances(ctx, nodeResourceGroup, *match.Name)
@@ -108,9 +139,8 @@ func (r *azureManagedMachinePoolReconciler) Reconcile(ctx context.Context, scope
 	}
 
 	var providerIDs = make([]string, len(instances))
-	for _, vm := range instances {
-		vm := vm
-		providerIDs = append(providerIDs, fmt.Sprintf("azure://%s", *vm.ID))
+	for i := 0; i < len(instances); i++ {
+		providerIDs[i] = fmt.Sprintf("azure://%s", *instances[i].ID)
 	}
 
 	scope.InfraMachinePool.Spec.ProviderIDList = providerIDs
@@ -136,3 +166,8 @@ func (r *azureManagedMachinePoolReconciler) Delete(ctx context.Context, scope *s
 
 	return nil
 }
+
+// IsAgentPoolVMSSNotFoundError returns true if the error is a AgentPoolVMSSNotFoundError
+func IsAgentPoolVMSSNotFoundError(err error) bool {
+	return errors.Is(err, notFoundErr)
+}