add taints to AzureManagedMachinePool

zmalik · zain.malik · commit 17be7481ed93 · 2022-02-15T10:15:07.000+01:00
diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go
@@ -599,6 +599,14 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane
 		agentPoolSpec.OSDiskSizeGB = *managedMachinePool.Spec.OSDiskSizeGB
 	}
 
+	if len(managedMachinePool.Spec.Taints) > 0 {
+		nodeTaints := make([]string, 0, len(managedMachinePool.Spec.Taints))
+		for _, t := range managedMachinePool.Spec.Taints {
+			nodeTaints = append(nodeTaints, fmt.Sprintf("%s=%s:%s", t.Key, t.Value, t.Effect))
+		}
+		agentPoolSpec.NodeTaints = nodeTaints
+	}
+
 	if managedMachinePool.Spec.Scaling != nil {
 		agentPoolSpec.EnableAutoScaling = to.BoolPtr(true)
 		agentPoolSpec.MaxCount = managedMachinePool.Spec.Scaling.MaxSize
diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go
@@ -562,6 +562,114 @@ func TestManagedControlPlaneScope_OSDiskType(t *testing.T) {
 	}
 }
 
+func TestManagedControlPlaneScope_Taints(t *testing.T) {
+	scheme := runtime.NewScheme()
+	_ = capiv1exp.AddToScheme(scheme)
+	_ = infrav1.AddToScheme(scheme)
+
+	cases := []struct {
+		Name     string
+		Input    ManagedControlPlaneScopeParams
+		Expected azure.AgentPoolSpec
+	}{
+		{
+			Name: "Without taints",
+			Input: ManagedControlPlaneScopeParams{
+				AzureClients: AzureClients{
+					Authorizer: autorest.NullAuthorizer{},
+				},
+				Cluster: &clusterv1.Cluster{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "cluster1",
+						Namespace: "default",
+					},
+				},
+				ControlPlane: &infrav1.AzureManagedControlPlane{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "cluster1",
+						Namespace: "default",
+					},
+					Spec: infrav1.AzureManagedControlPlaneSpec{
+						SubscriptionID: "00000000-0000-0000-0000-000000000000",
+					},
+				},
+				MachinePool:      getMachinePool("pool0"),
+				InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem),
+				PatchTarget:      getAzureMachinePool("pool0", infrav1.NodePoolModeSystem),
+			},
+			Expected: azure.AgentPoolSpec{
+
+				Name:         "pool0",
+				SKU:          "Standard_D2s_v3",
+				Replicas:     1,
+				Mode:         "System",
+				Cluster:      "cluster1",
+				VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/",
+			},
+		},
+		{
+			Name: "With taints",
+			Input: ManagedControlPlaneScopeParams{
+				AzureClients: AzureClients{
+					Authorizer: autorest.NullAuthorizer{},
+				},
+				Cluster: &clusterv1.Cluster{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "cluster1",
+						Namespace: "default",
+					},
+				},
+				ControlPlane: &infrav1.AzureManagedControlPlane{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "cluster1",
+						Namespace: "default",
+					},
+					Spec: infrav1.AzureManagedControlPlaneSpec{
+						SubscriptionID: "00000000-0000-0000-0000-000000000000",
+					},
+				},
+				MachinePool: getMachinePool("pool1"),
+				InfraMachinePool: getAzureMachinePoolWithTaints("pool1", infrav1.Taints{
+					infrav1.Taint{
+						Key:    "key1",
+						Value:  "value1",
+						Effect: "NoSchedule",
+					},
+				}),
+				PatchTarget: getAzureMachinePoolWithTaints("pool1", infrav1.Taints{
+					infrav1.Taint{
+						Key:    "key1",
+						Value:  "value1",
+						Effect: "NoSchedule",
+					},
+				}),
+			},
+			Expected: azure.AgentPoolSpec{
+				Name:         "pool1",
+				SKU:          "Standard_D2s_v3",
+				Mode:         "User",
+				Cluster:      "cluster1",
+				Replicas:     1,
+				NodeTaints:   []string{"key1=value1:NoSchedule"},
+				VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/",
+			},
+		},
+	}
+
+	for _, c := range cases {
+		c := c
+		t.Run(c.Name, func(t *testing.T) {
+			g := NewWithT(t)
+			fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.MachinePool, c.Input.InfraMachinePool, c.Input.ControlPlane).Build()
+			c.Input.Client = fakeClient
+			s, err := NewManagedControlPlaneScope(context.TODO(), c.Input)
+			g.Expect(err).To(Succeed())
+			agentPool := s.AgentPoolSpec()
+			g.Expect(agentPool).To(Equal(c.Expected))
+		})
+	}
+}
+
 func getAzureMachinePool(name string, mode infrav1.NodePoolMode) *infrav1.AzureManagedMachinePool {
 	return &infrav1.AzureManagedMachinePool{
 		ObjectMeta: metav1.ObjectMeta{
@@ -601,6 +709,12 @@ func getAzureMachinePoolWithMaxPods(name string, maxPods int32) *infrav1.AzureMa
 	return managedPool
 }
 
+func getAzureMachinePoolWithTaints(name string, taints infrav1.Taints) *infrav1.AzureManagedMachinePool {
+	managedPool := getAzureMachinePool(name, infrav1.NodePoolModeUser)
+	managedPool.Spec.Taints = taints
+	return managedPool
+}
+
 func getAzureMachinePoolWithOsDiskType(name string, osDiskType string) *infrav1.AzureManagedMachinePool {
 	managedPool := getAzureMachinePool(name, infrav1.NodePoolModeUser)
 	managedPool.Spec.OsDiskType = to.StringPtr(osDiskType)
diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go
@@ -78,6 +78,7 @@ func (s *Service) Reconcile(ctx context.Context) error {
 			EnableAutoScaling:   agentPoolSpec.EnableAutoScaling,
 			MaxCount:            agentPoolSpec.MaxCount,
 			MinCount:            agentPoolSpec.MinCount,
+			NodeTaints:          &agentPoolSpec.NodeTaints,
 			AvailabilityZones:   &agentPoolSpec.AvailabilityZones,
 			MaxPods:             agentPoolSpec.MaxPods,
 			OsDiskType:          containerservice.OSDiskType(to.String(agentPoolSpec.OsDiskType)),
diff --git a/azure/types.go b/azure/types.go
@@ -346,6 +346,9 @@ type AgentPoolSpec struct {
 	// Node labels - labels for all of the nodes present in node pool
 	NodeLabels map[string]*string `json:"nodeLabels,omitempty"`
 
+	// NodeTaints specifies the taints for nodes present in this agent pool.
+	NodeTaints []string `json:"nodeTaints,omitempty"`
+
 	// EnableAutoScaling - Whether to enable auto-scaler
 	EnableAutoScaling *bool `json:"enableAutoScaling,omitempty"`
 
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedmachinepools.yaml
@@ -257,6 +257,30 @@ spec:
               sku:
                 description: SKU is the size of the VMs in the node pool.
                 type: string
+              taints:
+                description: Taints specifies the taints for nodes present in this
+                  agent pool.
+                items:
+                  properties:
+                    effect:
+                      description: Effect specifies the effect for the taint
+                      enum:
+                      - NoSchedule
+                      - NoExecute
+                      - PreferNoSchedule
+                      type: string
+                    key:
+                      description: Key is the key of the taint
+                      type: string
+                    value:
+                      description: Value is the value of the taint
+                      type: string
+                  required:
+                  - effect
+                  - key
+                  - value
+                  type: object
+                type: array
             required:
             - mode
             - sku
diff --git a/docs/book/src/topics/managedcluster.md b/docs/book/src/topics/managedcluster.md
@@ -317,6 +317,27 @@ spec:
   osDiskType: "Ephemeral"
 ```
 
+### AKS Node Pool Taints
+
+You can configure the `Taints` value for each AKS node pool (`AzureManagedMachinePool`) that you define in your spec.
+
+Below is an example of `taints` configuration for the `agentpool0`:
+
+```
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureManagedMachinePool
+metadata:
+  name: agentpool0
+spec:
+  mode: System
+  osDiskSizeGB: 512
+  sku: Standard_D2s_v3
+  taints:
+    - effect: no-schedule
+      key: dedicated
+      value: kafka
+```
+
 ### Use a public Standard Load Balancer
 
 A public Load Balancer when integrated with AKS serves two purposes:
@@ -371,6 +392,32 @@ spec:
     enablePrivateClusterPublicFQDN: false # Allowed only when enablePrivateCluster is true
 ```
 
+## Immutable fields for Managed Clusters (AKS)
+
+Some fields from the family of Managed Clusters CRD are immutable. Which means 
+those can only be set during the creation time. 
+
+Following is the list of immutable fields for managed clusters:
+
+| CRD                      | jsonPath                           | Comment                   |
+|--------------------------|------------------------------------|---------------------------|
+| AzureManagedControlPlane | .spec.subscriptionID               |                           |
+| AzureManagedControlPlane | .spec.resourceGroupName            |                           |
+| AzureManagedControlPlane | .spec.nodeResourceGroupName        |                           |
+| AzureManagedControlPlane | .spec.location                     |                           |
+| AzureManagedControlPlane | .spec.sshPublicKey                 |                           |
+| AzureManagedControlPlane | .spec.dnsServiceIP                 |                           |
+| AzureManagedControlPlane | .spec.networkPlugin                |                           |
+| AzureManagedControlPlane | .spec.networkPolicy                |                           |
+| AzureManagedControlPlane | .spec.loadBalancerSKU              |                           |
+| AzureManagedControlPlane | .spec.apiServerAccessProfile       | except AuthorizedIPRanges |
+| AzureManagedMachinePool  | .spec.sku                          |                           |
+| AzureManagedMachinePool  | .spec.osDiskSizeGB                 |                           |
+| AzureManagedMachinePool  | .spec.osDiskType                   |                           |
+| AzureManagedMachinePool  | .spec.taints                       |                           |
+| AzureManagedMachinePool  | .spec.availabilityZones            |                           |
+| AzureManagedMachinePool  | .spec.maxPods                      |                           |
+
 ## Features
 
 AKS clusters deployed from CAPZ currently only support a limited,
diff --git a/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha3/azuremanagedmachinepool_conversion.go
@@ -38,6 +38,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 
 	dst.Spec.Name = restored.Spec.Name
 	dst.Spec.Scaling = restored.Spec.Scaling
+	dst.Spec.Taints = restored.Spec.Taints
 	dst.Spec.AvailabilityZones = restored.Spec.AvailabilityZones
 	dst.Spec.MaxPods = restored.Spec.MaxPods
 	dst.Spec.OsDiskType = restored.Spec.OsDiskType
diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go
diff --git a/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go b/exp/api/v1alpha4/azuremanagedmachinepool_conversion.go
@@ -38,6 +38,7 @@ func (src *AzureManagedMachinePool) ConvertTo(dstRaw conversion.Hub) error {
 
 	dst.Spec.Scaling = restored.Spec.Scaling
 	dst.Spec.Name = restored.Spec.Name
+	dst.Spec.Taints = restored.Spec.Taints
 	dst.Spec.AvailabilityZones = restored.Spec.AvailabilityZones
 	dst.Spec.MaxPods = restored.Spec.MaxPods
 	dst.Spec.OsDiskType = restored.Spec.OsDiskType
diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go
diff --git a/exp/api/v1beta1/azuremanagedmachinepool_types.go b/exp/api/v1beta1/azuremanagedmachinepool_types.go
@@ -64,6 +64,10 @@ type AzureManagedMachinePoolSpec struct {
 	// +optional
 	NodeLabels map[string]string `json:"nodeLabels,omitempty"`
 
+	// Taints specifies the taints for nodes present in this agent pool.
+	// +optional
+	Taints Taints `json:"taints,omitempty"`
+
 	// ProviderIDList is the unique identifier as specified by the cloud provider.
 	// +optional
 	ProviderIDList []string `json:"providerIDList,omitempty"`
@@ -89,6 +93,22 @@ type ManagedMachinePoolScaling struct {
 	MaxSize *int32 `json:"maxSize,omitempty"`
 }
 
+// TaintEffect is the effect for a Kubernetes taint.
+type TaintEffect string
+
+type Taint struct {
+	// Effect specifies the effect for the taint
+	// +kubebuilder:validation:Enum=NoSchedule;NoExecute;PreferNoSchedule
+	Effect TaintEffect `json:"effect"`
+	// Key is the key of the taint
+	Key string `json:"key"`
+	// Value is the value of the taint
+	Value string `json:"value"`
+}
+
+// Taints is an array of Taints.
+type Taints []Taint
+
 // AzureManagedMachinePoolStatus defines the observed state of AzureManagedMachinePool.
 type AzureManagedMachinePoolStatus struct {
 	// Ready is true when the provider resource is ready.
diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook.go
@@ -18,6 +18,7 @@ package v1beta1
 
 import (
 	"context"
+	"reflect"
 
 	"github.com/Azure/go-autorest/autorest/to"
 	"github.com/pkg/errors"
@@ -94,6 +95,14 @@ func (r *AzureManagedMachinePool) ValidateUpdate(oldRaw runtime.Object, client c
 		}
 	}
 
+	if !reflect.DeepEqual(r.Spec.Taints, old.Spec.Taints) {
+		allErrs = append(allErrs,
+			field.Invalid(
+				field.NewPath("Spec", "Taints"),
+				r.Spec.Taints,
+				"field is immutable"))
+	}
+
 	if !ensureStringSlicesAreEqual(r.Spec.AvailabilityZones, old.Spec.AvailabilityZones) {
 		allErrs = append(allErrs,
 			field.Invalid(
diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go
diff --git a/templates/test/ci/cluster-template-prow-aks-multi-tenancy.yaml b/templates/test/ci/cluster-template-prow-aks-multi-tenancy.yaml
diff --git a/templates/test/ci/prow-aks-multi-tenancy/patches/aks-agentpool1.yaml b/templates/test/ci/prow-aks-multi-tenancy/patches/aks-agentpool1.yaml
