AKS virtual network static placement for cross RG

snehala27 · LochanRn · commit e424fb4de0d2 · 2022-10-06T12:17:29.000+05:30
diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go
@@ -211,7 +211,7 @@ func (s *ManagedControlPlaneScope) Close(ctx context.Context) error {
 // Vnet returns the cluster Vnet.
 func (s *ManagedControlPlaneScope) Vnet() *infrav1.VnetSpec {
 	return &infrav1.VnetSpec{
-		ResourceGroup: s.ControlPlane.Spec.ResourceGroupName,
+		ResourceGroup: s.ControlPlane.Spec.VirtualNetwork.ResourceGroup,
 		Name:          s.ControlPlane.Spec.VirtualNetwork.Name,
 		VnetClassSpec: infrav1.VnetClassSpec{
 			CIDRBlocks: []string{s.ControlPlane.Spec.VirtualNetwork.CIDRBlock},
@@ -424,7 +424,7 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure
 		DNSServiceIP:      s.ControlPlane.Spec.DNSServiceIP,
 		VnetSubnetID: azure.SubnetID(
 			s.ControlPlane.Spec.SubscriptionID,
-			s.ControlPlane.Spec.ResourceGroupName,
+			s.VNetSpec().ResourceGroupName(),
 			s.ControlPlane.Spec.VirtualNetwork.Name,
 			s.ControlPlane.Spec.VirtualNetwork.Subnet.Name,
 		),
diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go
@@ -165,7 +165,7 @@ func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane
 		OSType:        managedMachinePool.Spec.OSType,
 		VnetSubnetID: azure.SubnetID(
 			managedControlPlane.Spec.SubscriptionID,
-			managedControlPlane.Spec.ResourceGroupName,
+			managedControlPlane.Spec.VirtualNetwork.ResourceGroup,
 			managedControlPlane.Spec.VirtualNetwork.Name,
 			managedControlPlane.Spec.VirtualNetwork.Subnet.Name,
 		),
diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml
@@ -733,6 +733,10 @@ spec:
                     type: string
                   name:
                     type: string
+                  resourceGroup:
+                    description: ResourceGroup is the name of the Azure resource group
+                      for the VNet and Subnet.
+                    type: string
                   subnet:
                     description: ManagedControlPlaneSubnet describes a subnet for
                       an AKS cluster.
diff --git a/docs/book/src/topics/managedcluster.md b/docs/book/src/topics/managedcluster.md
@@ -175,6 +175,29 @@ should be fairly clear from context.
 | networkPolicy             | azure, calico                 |
 
 
+### Use an existing Virtual Network to provision an AKS cluster.
+
+If you'd like to deploy your AKS cluster in an existing Virtual Network, but create the cluster itself in a different resource group, you can configure the AzureManagedControlPlane resource with a reference to the existing Virtual Network and subnet. For example:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureManagedControlPlane
+metadata:
+  name: my-cluster-control-plane
+spec:
+  location: southcentralus
+  resourceGroupName: foo-bar
+  sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""}
+  subscriptionID: 00000000-0000-0000-0000-000000000000 # fake uuid
+  version: v1.21.2
+  virtualNetwork:
+    cidrBlock: 10.0.0.0/8
+    name: test-vnet
+    resourceGroup: test-rg
+    subnet:
+      cidrBlock: 10.0.2.0/24
+      name: test-subnet  
+```
 ### Multitenancy
 
 Multitenancy for managed clusters can be configured by using `aks-multi-tenancy` flavor. The steps for creating an azure managed identity and mapping it to an `AzureClusterIdentity` are similar to the ones described [here](https://capz.sigs.k8s.io/topics/multitenancy.html).
@@ -451,6 +474,7 @@ Following is the list of immutable fields for managed clusters:
 | AzureManagedControlPlane  | .spec.networkPolicy          |                           |
 | AzureManagedControlPlane  | .spec.loadBalancerSKU        |                           |
 | AzureManagedControlPlane  | .spec.apiServerAccessProfile | except AuthorizedIPRanges |
+| AzureManagedControlPlane  | .spec.virtualNetwork         |                           |
 | AzureManagedMachinePool   | .spec.sku                    |                           |
 | AzureManagedMachinePool   | .spec.osDiskSizeGB           |                           |
 | AzureManagedMachinePool   | .spec.osDiskType             |                           |
diff --git a/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go b/exp/api/v1alpha3/azuremanagedcontrolplane_conversion.go
@@ -41,6 +41,7 @@ func (src *AzureManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.LoadBalancerProfile = restored.Spec.LoadBalancerProfile
 	dst.Spec.APIServerAccessProfile = restored.Spec.APIServerAccessProfile
 	dst.Spec.AddonProfiles = restored.Spec.AddonProfiles
+	dst.Spec.VirtualNetwork.ResourceGroup = restored.Spec.VirtualNetwork.ResourceGroup
 
 	dst.Status.LongRunningOperationStates = restored.Status.LongRunningOperationStates
 	dst.Status.Conditions = restored.Status.Conditions
@@ -81,3 +82,13 @@ func (dst *AzureManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) erro
 	src := srcRaw.(*infrav1exp.AzureManagedControlPlaneList)
 	return Convert_v1beta1_AzureManagedControlPlaneList_To_v1alpha3_AzureManagedControlPlaneList(src, dst, nil)
 }
+
+// Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork converts v1beta1 ManagedControlPlaneVirtualNetwork to v1alpha3 ManagedControlPlaneVirtualNetwork.
+func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha3_ManagedControlPlaneVirtualNetwork(in *infrav1exp.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s apiconversion.Scope) error {
+	out.Name = in.Name
+	out.Subnet.Name = in.Subnet.Name
+	out.Subnet.CIDRBlock = in.Subnet.CIDRBlock
+	out.CIDRBlock = in.CIDRBlock
+
+	return nil
+}
diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go
diff --git a/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go b/exp/api/v1alpha4/azuremanagedcontrolplane_conversion.go
@@ -38,6 +38,7 @@ func (src *AzureManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 
 	dst.Spec.AddonProfiles = restored.Spec.AddonProfiles
 	dst.Status.Conditions = restored.Status.Conditions
+	dst.Spec.VirtualNetwork.ResourceGroup = restored.Spec.VirtualNetwork.ResourceGroup
 
 	return nil
 }
@@ -74,3 +75,12 @@ func (dst *AzureManagedControlPlaneList) ConvertFrom(srcRaw conversion.Hub) erro
 	src := srcRaw.(*infrav1exp.AzureManagedControlPlaneList)
 	return Convert_v1beta1_AzureManagedControlPlaneList_To_v1alpha4_AzureManagedControlPlaneList(src, dst, nil)
 }
+
+// Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha4_ManagedControlPlaneVirtualNetwork converts v1beta1 ManagedControlPlaneVirtualNetwork to v1alpha4 ManagedControlPlaneVirtualNetwork.
+func Convert_v1beta1_ManagedControlPlaneVirtualNetwork_To_v1alpha4_ManagedControlPlaneVirtualNetwork(in *infrav1exp.ManagedControlPlaneVirtualNetwork, out *ManagedControlPlaneVirtualNetwork, s apiconversion.Scope) error {
+	out.Name = in.Name
+	out.Subnet.Name = in.Subnet.Name
+	out.Subnet.CIDRBlock = in.Subnet.CIDRBlock
+	out.CIDRBlock = in.CIDRBlock
+	return nil
+}
diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go
diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_default.go b/exp/api/v1beta1/azuremanagedcontrolplane_default.go
@@ -60,6 +60,9 @@ func (m *AzureManagedControlPlane) setDefaultVirtualNetwork() {
 	if m.Spec.VirtualNetwork.CIDRBlock == "" {
 		m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDR
 	}
+	if m.Spec.VirtualNetwork.ResourceGroup == "" {
+		m.Spec.VirtualNetwork.ResourceGroup = m.Spec.ResourceGroupName
+	}
 }
 
 // setDefaultSubnet sets the default Subnet for an AzureManagedControlPlane.
diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go
@@ -206,6 +206,9 @@ type ManagedControlPlaneVirtualNetwork struct {
 	CIDRBlock string `json:"cidrBlock"`
 	// +optional
 	Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"`
+	// ResourceGroup is the name of the Azure resource group for the VNet and Subnet.
+	// +optional
+	ResourceGroup string `json:"resourceGroup,omitempty"`
 }
 
 // ManagedControlPlaneSubnet describes a subnet for an AKS cluster.
diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go
@@ -240,6 +240,10 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client
 		}
 	}
 
+	if errs := m.validateVirtualNetowrkUpdate(old); len(errs) > 0 {
+		allErrs = append(allErrs, errs...)
+	}
+
 	if errs := m.validateAPIServerAccessProfileUpdate(old); len(errs) > 0 {
 		allErrs = append(allErrs, errs...)
 	}
@@ -471,6 +475,51 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileUpdate(old *Azu
 	return allErrs
 }
 
+// validateVirtualNetowrkUpdate validates update to APIServerAccessProfile.
+func (m *AzureManagedControlPlane) validateVirtualNetowrkUpdate(old *AzureManagedControlPlane) field.ErrorList {
+	var allErrs field.ErrorList
+	if old.Spec.VirtualNetwork.Name != m.Spec.VirtualNetwork.Name {
+		allErrs = append(allErrs,
+			field.Invalid(
+				field.NewPath("Spec", "VirtualNetwork.Name"),
+				m.Spec.VirtualNetwork.Name,
+				"Virtual Network Name is immutable"))
+	}
+
+	if old.Spec.VirtualNetwork.CIDRBlock != m.Spec.VirtualNetwork.CIDRBlock {
+		allErrs = append(allErrs,
+			field.Invalid(
+				field.NewPath("Spec", "VirtualNetwork.CIDRBlock"),
+				m.Spec.VirtualNetwork.CIDRBlock,
+				"Virtual Network CIDRBlock is immutable"))
+	}
+
+	if old.Spec.VirtualNetwork.Subnet.Name != m.Spec.VirtualNetwork.Subnet.Name {
+		allErrs = append(allErrs,
+			field.Invalid(
+				field.NewPath("Spec", "VirtualNetwork.Subnet.Name"),
+				m.Spec.VirtualNetwork.Subnet.Name,
+				"Subnet Name is immutable"))
+	}
+
+	if old.Spec.VirtualNetwork.Subnet.CIDRBlock != m.Spec.VirtualNetwork.Subnet.CIDRBlock {
+		allErrs = append(allErrs,
+			field.Invalid(
+				field.NewPath("Spec", "VirtualNetwork.Subnet.CIDRBlock"),
+				m.Spec.VirtualNetwork.Subnet.CIDRBlock,
+				"Subnet CIDRBlock is immutable"))
+	}
+
+	if old.Spec.VirtualNetwork.ResourceGroup != m.Spec.VirtualNetwork.ResourceGroup {
+		allErrs = append(allErrs,
+			field.Invalid(
+				field.NewPath("Spec", "VirtualNetwork.ResourceGroup"),
+				m.Spec.VirtualNetwork.ResourceGroup,
+				"Virtual Network Resource Group is immutable"))
+	}
+	return allErrs
+}
+
 func (m *AzureManagedControlPlane) validateName(_ client.Client) error {
 	if lName := strings.ToLower(m.Name); strings.Contains(lName, "microsoft") ||
 		strings.Contains(lName, "windows") {
diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go
@@ -777,6 +777,108 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "AzureManagedControlPlane.VirtualNetwork Name is mutable",
+			oldAMCP: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					DNSServiceIP: to.StringPtr("192.168.0.0"),
+					Version:      "v1.18.0",
+					VirtualNetwork: ManagedControlPlaneVirtualNetwork{
+						Name:          "test-network",
+						CIDRBlock:     "10.0.0.0/8",
+						ResourceGroup: "test-rg",
+						Subnet: ManagedControlPlaneSubnet{
+							Name:      "test-subnet",
+							CIDRBlock: "10.0.2.0/24",
+						},
+					},
+				},
+			},
+			amcp: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					DNSServiceIP: to.StringPtr("192.168.0.0"),
+					Version:      "v1.18.0",
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "AzureManagedControlPlane.VirtualNetwork Name is mutable",
+			oldAMCP: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					DNSServiceIP: to.StringPtr("192.168.0.0"),
+					Version:      "v1.18.0",
+				},
+			},
+			amcp: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					DNSServiceIP: to.StringPtr("192.168.0.0"),
+					Version:      "v1.18.0",
+					VirtualNetwork: ManagedControlPlaneVirtualNetwork{
+						Name:          "test-network",
+						CIDRBlock:     "10.0.0.0/8",
+						ResourceGroup: "test-rg",
+						Subnet: ManagedControlPlaneSubnet{
+							Name:      "test-subnet",
+							CIDRBlock: "10.0.2.0/24",
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+		{
+			name: "AzureManagedControlPlane.VirtualNetwork Name is mutable",
+			oldAMCP: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					DNSServiceIP: to.StringPtr("192.168.0.0"),
+					Version:      "v1.18.0",
+					VirtualNetwork: ManagedControlPlaneVirtualNetwork{
+						Name:          "test-network",
+						CIDRBlock:     "10.0.0.0/8",
+						ResourceGroup: "test-rg",
+						Subnet: ManagedControlPlaneSubnet{
+							Name:      "test-subnet",
+							CIDRBlock: "10.0.2.0/24",
+						},
+					},
+				},
+			},
+			amcp: &AzureManagedControlPlane{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-cluster",
+				},
+				Spec: AzureManagedControlPlaneSpec{
+					DNSServiceIP: to.StringPtr("192.168.0.0"),
+					Version:      "v1.18.0",
+					VirtualNetwork: ManagedControlPlaneVirtualNetwork{
+						Name:          "test-network",
+						CIDRBlock:     "10.0.0.0/8",
+						ResourceGroup: "test-rg",
+						Subnet: ManagedControlPlaneSubnet{
+							Name:      "test-subnet",
+							CIDRBlock: "10.0.2.0/24",
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {

Original file line number	Diff line number	Diff line change
`@@ -60,6 +60,9 @@ func (m *AzureManagedControlPlane) setDefaultVirtualNetwork() {`
`60`	`60`	`if m.Spec.VirtualNetwork.CIDRBlock == "" {`
`61`	`61`	`m.Spec.VirtualNetwork.CIDRBlock = defaultAKSVnetCIDR`
`62`	`62`	`}`
	`63`	`+ if m.Spec.VirtualNetwork.ResourceGroup == "" {`
	`64`	`+ m.Spec.VirtualNetwork.ResourceGroup = m.Spec.ResourceGroupName`
	`65`	`+ }`
`63`	`66`	`}`
`64`	`67`
`65`	`68`	`// setDefaultSubnet sets the default Subnet for an AzureManagedControlPlane.`
Original file line number	Diff line number	Diff line change
`@@ -206,6 +206,9 @@ type ManagedControlPlaneVirtualNetwork struct {`
`206`	`206`	CIDRBlock string `json:"cidrBlock"`
`207`	`207`	`// +optional`
`208`	`208`	Subnet ManagedControlPlaneSubnet `json:"subnet,omitempty"`
	`209`	`+ // ResourceGroup is the name of the Azure resource group for the VNet and Subnet.`
	`210`	`+ // +optional`
	`211`	+ ResourceGroup string `json:"resourceGroup,omitempty"`
`209`	`212`	`}`
`210`	`213`
`211`	`214`	`// ManagedControlPlaneSubnet describes a subnet for an AKS cluster.`