feat!: (IAC-1009) Add support for network plugin mode overlay (#360)

Author: riragh

docs/CONFIG-VARS.md

@@ -102,8 +102,8 @@ az vm image terms accept --urn Canonical:0001-com-ubuntu-pro-focal-fips:pro-fips
 | subnets | Subnets to be created and their settings | map(object) | *check below* | This variable is ignored when subnet_names is set (AKA bring your own subnets). All defined subnets must exist within the vnet address space. |
 | cluster_egress_type | The outbound (egress) routing method to be used for this Kubernetes Cluster | string | "loadBalancer" | Possible values: <ul><li>`loadBalancer`<li>`userDefinedRouting`</ul> By default, AKS will create and use a [loadbalancer](https://docs.microsoft.com/en-us/azure/aks/load-balancer-standard) for outgoing connections.<p>Set to `userDefinedRouting` when using your own network [egress](https://docs.microsoft.com/en-us/azure/aks/egress-outboundtype).|
 | aks_network_plugin | Network plugin to use for networking. Currently supported values are `azure` and `kubenet`| string | `kubenet`| For details see Azure's documentation on: [configure kubenet](https://docs.microsoft.com/en-us/azure/aks/configure-kubenet), [Configure Azure CNI](https://learn.microsoft.com/en-us/azure/aks/configure-azure-cni).<br>**Note**: To support Azure CNI your Subnet must be large enough to accommodate the nodes, pods, and all Kubernetes and Azure resources that might be provisioned in your cluster.<br>To calculate the minimum subnet size including an additional node for upgrade operations use formula: `(number of nodes + 1) + ((number of nodes + 1) * maximum pods per node that you configure)` <br>Example for a 5 node cluster: `(5) + (5 * 110) = 555 (/22 or larger)`|
-| aks_network_policy | Sets up network policy to be used with Azure CNI. Network policy allows to control the traffic flow between pods. Currently supported values are `calico` and `azure`.| string | `azure`| Network policy `azure` is only supported for `aks_network_plugin = azure` and network policy `calico` is supported for both `aks_network_plugin` values `azure` and `kubenet`. |
-
+| aks_network_policy | Sets up network policy to be used with Azure CNI. Network policy allows to control the traffic flow between pods. Currently supported values are `calico` and `azure`.| string | null | Network policy `azure` is only supported for `aks_network_plugin = azure` and network policy `calico` is supported for both `aks_network_plugin` values `azure` and `kubenet`. |
+| aks_network_plugin_mode | Specifies the network plugin mode used for building the Kubernetes network. Possible value is `overlay`.| string | null | When `aks_network_plugin_mode` is set to `overlay` , the `aks_network_plugin` field can only be set to `azure`. For details see Azure's documentation on: [Configure Azure CNI Overlay networking](https://learn.microsoft.com/en-us/azure/aks/azure-cni-overlay).|
 
 The default values for the `subnets` variable are as follows:
 

main.tf

@@ -153,6 +153,7 @@ module "aks" {
   aks_log_analytics_workspace_id           = var.create_aks_azure_monitor ? azurerm_log_analytics_workspace.viya4[0].id : null
   aks_network_plugin                       = var.aks_network_plugin
   aks_network_policy                       = var.aks_network_policy
+  aks_network_plugin_mode                  = var.aks_network_plugin_mode
   aks_dns_service_ip                       = var.aks_dns_service_ip
   aks_docker_bridge_cidr                   = var.aks_docker_bridge_cidr
   cluster_egress_type                      = local.cluster_egress_type

modules/azure_aks/main.tf

@@ -21,9 +21,6 @@ resource "azurerm_kubernetes_cluster" "aks" {
   private_dns_zone_id             = var.aks_private_cluster && var.aks_cluster_private_dns_zone_id != "" ? var.aks_cluster_private_dns_zone_id : (var.aks_private_cluster ? "System" : null)
 
   network_profile {
-    network_plugin = var.aks_network_plugin
-    network_policy = var.aks_network_plugin == "kubenet" && var.aks_network_policy == "azure" ? null : var.aks_network_policy
-
     # Docs on AKS Advanced Networking config
     # https://docs.microsoft.com/en-us/azure/architecture/aws-professional/networking
     # https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-vnet-plan-design-arm
@@ -32,12 +29,15 @@ resource "azurerm_kubernetes_cluster" "aks" {
     # https://docs.microsoft.com/en-us/azure/aks/load-balancer-standard
     # https://docs.microsoft.com/en-us/azure/aks/egress-outboundtype
 
-    service_cidr       = var.aks_service_cidr
-    dns_service_ip     = var.aks_dns_service_ip
-    pod_cidr           = var.aks_network_plugin == "kubenet" ? var.aks_pod_cidr : null
-    docker_bridge_cidr = var.aks_docker_bridge_cidr
-    outbound_type      = var.cluster_egress_type
-    load_balancer_sku  = "standard"
+    network_plugin      = var.aks_network_plugin
+    network_policy      = var.aks_network_policy
+    network_plugin_mode = var.aks_network_plugin_mode
+    service_cidr        = var.aks_service_cidr
+    dns_service_ip      = var.aks_dns_service_ip
+    pod_cidr            = var.aks_network_plugin == "kubenet" ? var.aks_pod_cidr : null
+    docker_bridge_cidr  = var.aks_docker_bridge_cidr
+    outbound_type       = var.cluster_egress_type
+    load_balancer_sku   = "standard"
   }
 
   dynamic "linux_profile" {
@@ -102,6 +102,14 @@ resource "azurerm_kubernetes_cluster" "aks" {
 
   lifecycle {
     ignore_changes = [default_node_pool[0].node_count]
+    precondition {
+      condition     = var.aks_network_policy != "azure" || var.aks_network_plugin == "azure"
+      error_message = "When aks_network_policy is set to `azure`, the aks_network_plugin field can only be set to `azure`."
+    }
+    precondition {
+      condition     = var.aks_network_plugin_mode != "overlay" || var.aks_network_plugin == "azure"
+      error_message = "When network_plugin_mode is set to `overlay`, the aks_network_plugin field can only be set to `azure`."
+    }
   }
 
   tags = var.aks_cluster_tags

modules/azure_aks/variables.tf

@@ -133,7 +133,13 @@ variable "aks_network_plugin" {
 variable "aks_network_policy" {
   description = "Sets up network policy to be used with Azure CNI. Network policy allows us to control the traffic flow between pods. Currently supported values are calico and azure. Changing this forces a new resource to be created."
   type        = string
-  default     = "azure"
+  default     = null
+}
+
+variable "aks_network_plugin_mode" {
+  description = "Specifies the network plugin mode used for building the Kubernetes network. Possible value is `overlay`. Changing this forces a new resource to be created."
+  type        = string
+  default     = null
 }
 
 variable "aks_dns_service_ip" {

variables.tf

@@ -169,12 +169,13 @@ variable "aks_network_plugin" {
 variable "aks_network_policy" {
   description = "Sets up network policy to be used with Azure CNI. Network policy allows control of the traffic flow between pods. Currently supported values are calico and azure. Changing this forces a new resource to be created."
   type        = string
-  default     = "azure"
+  default     = null
+}
 
-  validation {
-    condition     = contains(["azure", "calico"], var.aks_network_policy)
-    error_message = "Error: Currently the supported values are 'calico' and 'azure'."
-  }
+variable "aks_network_plugin_mode" {
+  description = "Specifies the network plugin mode used for building the Kubernetes network. Possible value is `overlay`. Changing this forces a new resource to be created."
+  type        = string
+  default     = null
 }
 
 variable "aks_dns_service_ip" {