Support for VCN Native Pod Networking (#542)

* feat: added support for Native Pod Networking

Signed-off-by: Ali Mukadam <[email protected]>

* fix: incorrect value for flannel

Signed-off-by: Ali Mukadam <[email protected]>

* feat: added support for VCN Native pod networking. Closes #539

Signed-off-by: Ali Mukadam <[email protected]>

* fix: removed commented code as requested by @Djelibeybi

Signed-off-by: Ali Mukadam <[email protected]>

Author: hyder

docs/terraformoptions.adoc

@@ -223,6 +223,8 @@ EOT
     int_lb   = { netnum = 16, newbits = 11 }
     pub_lb   = { netnum = 17, newbits = 11 }
     workers  = { netnum = 1, newbits = 2 }
+    pods     = { netnum = 2, newbits = 2 }
+    fss      = { netnum = 18, newbits = 11 }
   }
 |
   subnets = {
@@ -232,6 +234,8 @@ EOT
     int_lb   = { netnum = 16, newbits = 11 }
     pub_lb   = { netnum = 17, newbits = 11 }
     workers  = { netnum = 1, newbits = 2 }
+    pods     = { netnum = 2, newbits = 2 }
+    fss      = { netnum = 18, newbits = 11 }
   }
 
 
@@ -251,6 +255,7 @@ EOT
 |oke-vcn
 
 |===
+
 == Reuse VCN
 
 [stripes=odd,cols="1m,4d,2m,2m", options=header,width="100%"]
@@ -596,6 +601,11 @@ EOT
 |true/false
 |false
 
+|allow_pod_internet_access
+|Allow pods to egress to internet. Required if the pods are invoking Internet services.
+|true/false
+|true
+
 |allow_worker_internet_access
 |Whether to allow access to NodePorts when worker nodes are deployed in public mode..
 |true/false
@@ -611,6 +621,11 @@ EOT
 |
 |oke
 
+|cni_type
+|The CNI for the cluster. Choose between flannel or npn (Native Pod Networking).
+|flannel/npn
+|flannel
+
 |control_plane_type
 |Whether to allow public or private access to the control plane endpoint.
 |public/private
@@ -710,6 +725,11 @@ EOT
 |one,all,none
 |none
 
+|max_pods_per_node
+|The maximum number of pods to deploy per node. Absolute maximum is 110. Applies only when CNI type is `npn`.
+|
+|31
+
 |node_pools
 |The number, shape of node pools and node_pool_size to create. Each key and tuple pair corresponds to 1 node pool.
 

main.tf

@@ -192,12 +192,14 @@ module "network" {
 
 
   # control plane endpoint parameters
+  cni_type                    = var.cni_type
   control_plane_type          = var.control_plane_type
   control_plane_allowed_cidrs = var.control_plane_allowed_cidrs
 
   # oke worker network parameters
   allow_node_port_access       = var.allow_node_port_access
   allow_worker_internet_access = var.allow_worker_internet_access
+  allow_pod_internet_access    = var.allow_pod_internet_access
   allow_worker_ssh_access      = var.allow_worker_ssh_access
   worker_type                  = var.worker_type
 
@@ -247,6 +249,7 @@ module "oke" {
   cluster_options_kubernetes_network_config_pods_cidr     = var.pods_cidr
   cluster_options_kubernetes_network_config_services_cidr = var.services_cidr
   cluster_subnets                                         = module.network.subnet_ids
+  cni_type                                                = var.cni_type
   vcn_id                                                  = local.vcn_id
   use_cluster_encryption                                  = var.use_cluster_encryption
   cluster_kms_key_id                                      = var.cluster_kms_key_id
@@ -256,6 +259,7 @@ module "oke" {
   admission_controller_options                            = var.admission_controller_options
 
   # oke node pool parameters
+  max_pods_per_node               = var.max_pods_per_node
   node_pools                      = var.node_pools
   node_pool_name_prefix           = var.node_pool_name_prefix
   node_pool_image_id              = var.node_pool_image_id
@@ -272,7 +276,8 @@ module "oke" {
   # oke load balancer parameters
   preferred_load_balancer = var.preferred_load_balancer
 
-  # worker nsgs
+  # nsgs
+  pod_nsgs = concat(module.network.pod_nsg_id)
   worker_nsgs = concat(var.worker_nsgs, [module.network.worker_nsg_id])
 
   freeform_tags = var.freeform_tags["oke"]

modules/network/locals.tf

@@ -20,6 +20,8 @@ locals {
 
   workers_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["workers"], "newbits"), lookup(var.subnets["workers"], "netnum"))
 
+  pods_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["pods"], "newbits"), lookup(var.subnets["pods"], "netnum"))
+
   fss_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["fss"], "newbits"), lookup(var.subnets["fss"], "netnum"))
 
   anywhere = "0.0.0.0/0"
@@ -91,15 +93,15 @@ locals {
       destination      = local.osn,
       destination_type = "SERVICE_CIDR_BLOCK",
       protocol         = local.tcp_protocol,
-      port             = 443,
+      port             = -1,
       stateless        = false
     },
     {
-      description      = "Allow all TCP traffic from control plane to worker nodes",
+      description      = "Allow Kubernetes Control plane to communicate with worker nodes",
       destination      = local.workers_subnet,
       destination_type = "CIDR_BLOCK",
       protocol         = local.tcp_protocol,
-      port             = -1,
+      port             = 10250,
       stateless        = false
     },
     {
@@ -149,17 +151,9 @@ locals {
 
   # workers
   workers_egress = [
-    {
-      description      = "Allow egress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet",
-      destination      = local.workers_subnet,
-      destination_type = "CIDR_BLOCK",
-      protocol         = local.all_protocols,
-      port             = -1,
-      stateless        = false
-    },
     {
       description      = "Allow ICMP traffic for path discovery",
-      destination      = local.workers_subnet
+      destination      = local.anywhere
       destination_type = "CIDR_BLOCK",
       protocol         = local.icmp_protocol,
       port             = -1,
@@ -203,7 +197,7 @@ locals {
     {
       description = "Allow control plane to communicate with worker nodes",
       protocol    = local.tcp_protocol,
-      port        = -1,
+      port        = 10250,
       source      = local.cp_subnet,
       source_type = "CIDR_BLOCK",
       stateless   = false
@@ -219,6 +213,60 @@ locals {
     }
   ]
 
+  pods_egress = [
+    {
+      description      = "Allow pods to communicate with other pods.",
+      destination      = local.pods_subnet,
+      destination_type = "CIDR_BLOCK",
+      protocol         = local.all_protocols,
+      port             = -1,
+      stateless        = false
+    },
+    {
+      description      = "Allow ICMP traffic for path discovery",
+      destination      = local.osn,
+      destination_type = "SERVICE_CIDR_BLOCK",
+      protocol         = local.icmp_protocol,
+      port             = -1,
+      stateless        = false
+    },
+    {
+      description      = "Allow pods to communicate with OCI Services",
+      destination      = local.osn,
+      destination_type = "SERVICE_CIDR_BLOCK",
+      protocol         = local.tcp_protocol,
+      port             = -1,
+      stateless        = false
+    },
+  ]
+
+  pods_ingress = [
+    {
+      description = "Allow worker nodes to access pods.",
+      protocol    = local.all_protocols,
+      port        = -1,
+      source      = local.cp_subnet,
+      source_type = "CIDR_BLOCK",
+      stateless   = false
+    },
+    {
+      description = "Allow Kubernetes Control Plane to communicate with pods.",
+      protocol    = local.all_protocols,
+      port        = -1,
+      source      = local.workers_subnet,
+      source_type = "CIDR_BLOCK",
+      stateless   = false
+    },
+    {
+      description = "Allow pods to communicate with each other.",
+      protocol    = local.all_protocols,
+      port        = -1,
+      source      = local.pods_subnet,
+      source_type = "CIDR_BLOCK",
+      stateless   = false
+    },
+  ]
+
   int_lb_egress = [
     {
       description      = "Allow stateful egress to workers. Required for NodePorts",
@@ -247,7 +295,7 @@ locals {
   ]
 
   # Combine supplied allow list and the public load balancer subnet
-  internal_lb_allowed_cidrs = var.load_balancers == "both"? concat(var.internal_lb_allowed_cidrs, tolist([local.pub_lb_subnet])) : var.internal_lb_allowed_cidrs
+  internal_lb_allowed_cidrs = var.load_balancers == "both" ? concat(var.internal_lb_allowed_cidrs, tolist([local.pub_lb_subnet])) : var.internal_lb_allowed_cidrs
 
   # Create a Cartesian product of allowed cidrs and ports
   internal_lb_allowed_cidrs_and_ports = setproduct(local.internal_lb_allowed_cidrs, var.internal_lb_allowed_ports)

modules/network/nsgs.tf

@@ -37,6 +37,19 @@ resource "oci_core_network_security_group_security_rule" "cp_egress" {
   }
 
   count = length(local.cp_egress)
+}
+
+resource "oci_core_network_security_group_security_rule" "cp_egress_npn" {
+  network_security_group_id = oci_core_network_security_group.cp.id
+  description               = "Allow Kubernetes Control plane to communicate with pods"
+  destination               = local.pods_subnet
+  destination_type          = "CIDR_BLOCK"
+  direction                 = "EGRESS"
+  protocol                  = local.all_protocols
+
+  stateless = false
+
+  count = var.cni_type == "npn" ? 1 :0
 
 }
 
@@ -134,7 +147,32 @@ resource "oci_core_network_security_group_security_rule" "workers_egress" {
   }
 
   count = length(local.workers_egress)
+}
 
+resource "oci_core_network_security_group_security_rule" "workers_egress_flannel" {
+  network_security_group_id = oci_core_network_security_group.workers.id
+  description               = "Allow egress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet"
+  destination               = local.workers_subnet
+  destination_type          = "CIDR_BLOCK"
+  direction                 = "EGRESS"
+  protocol                  = local.all_protocols
+
+  stateless = false
+
+  count = var.cni_type == "flannel" ? 1: 0
+}
+
+resource "oci_core_network_security_group_security_rule" "workers_egress_npn" {
+  network_security_group_id = oci_core_network_security_group.workers.id
+  description               = "Allow worker nodes access to pods"
+  destination               = local.pods_subnet
+  destination_type          = "CIDR_BLOCK"
+  direction                 = "EGRESS"
+  protocol                  = local.all_protocols
+
+  stateless = false
+
+  count = var.cni_type == "npn" ? 1: 0
 }
 
 # add this rule separately so it can be controlled independently
@@ -290,6 +328,58 @@ resource "oci_core_network_security_group_security_rule" "workers_ssh_ingress_fr
 
 }
 
+# pod nsg and rules
+resource "oci_core_network_security_group" "pods" {
+  compartment_id = var.compartment_id
+  display_name   = var.label_prefix == "none" ? "pods" : "${var.label_prefix}-pods"
+  vcn_id         = var.vcn_id
+}
+
+resource "oci_core_network_security_group_security_rule" "pods_egress" {
+  network_security_group_id = oci_core_network_security_group.pods.id
+  description               = local.pods_egress[count.index].description
+  destination               = local.pods_egress[count.index].destination
+  destination_type          = local.pods_egress[count.index].destination_type
+  direction                 = "EGRESS"
+  protocol                  = local.pods_egress[count.index].protocol
+
+  stateless = false
+
+  dynamic "tcp_options" {
+    for_each = local.pods_egress[count.index].protocol == local.tcp_protocol && local.pods_egress[count.index].port != -1 ? [1] : []
+    content {
+      destination_port_range {
+        min = local.pods_egress[count.index].port
+        max = local.pods_egress[count.index].port
+      }
+    }
+  }
+
+  dynamic "icmp_options" {
+    for_each = local.pods_egress[count.index].protocol == local.icmp_protocol ? [1] : []
+    content {
+      type = 3
+      code = 4
+    }
+  }
+
+  count = var.cni_type =="npn" ? length(local.pods_egress) : 0
+}
+
+# add this rule separately so it can be controlled independently
+resource "oci_core_network_security_group_security_rule" "pods_egress_internet" {
+  network_security_group_id = oci_core_network_security_group.pods.id
+  description               = "Allow pods access to Internet"
+  destination               = local.anywhere
+  destination_type          = "CIDR_BLOCK"
+  direction                 = "EGRESS"
+  protocol                  = local.tcp_protocol
+
+  stateless = false
+  count = (var.cni_type =="npn" && var.allow_pod_internet_access == true) ? 1 : 0
+
+}
+
 # internal lb nsg and rules
 resource "oci_core_network_security_group" "int_lb" {
   compartment_id = var.compartment_id

modules/network/outputs.tf

@@ -4,6 +4,7 @@
 output "subnet_ids" {
   value = {
     "cp"      = join(",", oci_core_subnet.cp[*].id)
+    "pods" = join(",", oci_core_subnet.pods[*].id)
     "workers" = join(",", oci_core_subnet.workers[*].id)
     "int_lb"  = join(",", oci_core_subnet.int_lb[*].id)
     "pub_lb"  = join(",", oci_core_subnet.pub_lb[*].id)
@@ -22,6 +23,10 @@ output "pub_lb" {
   value = var.load_balancers == "public" || var.load_balancers == "both" ? oci_core_network_security_group.pub_lb[0].id : ""
 }
 
+output "pod_nsg_id" {
+  value = tolist([oci_core_network_security_group.pods.id])
+}
+
 output "worker_nsg_id" {
   value = oci_core_network_security_group.workers.id
 }

modules/network/subnets.tf

@@ -22,6 +22,18 @@ resource "oci_core_subnet" "workers" {
   vcn_id                     = var.vcn_id
 }
 
+resource "oci_core_subnet" "pods" {
+  cidr_block                 = local.pods_subnet
+  compartment_id             = var.compartment_id
+  display_name               = var.label_prefix == "none" ? "pods" : "${var.label_prefix}-pods"
+  dns_label                  = "pods"
+  prohibit_public_ip_on_vnic = true
+  route_table_id             = var.nat_route_id
+  vcn_id                     = var.vcn_id
+
+  count = var.cni_type == "npn" ? 1 : 0
+}
+
 resource "oci_core_subnet" "int_lb" {
   cidr_block                 = local.int_lb_subnet
   compartment_id             = var.compartment_id

modules/network/variables.tf

@@ -19,6 +19,8 @@ variable "vcn_id" {}
 
 # cluster endpoint
 
+variable "cni_type" {}
+
 variable "control_plane_type" {
   type = string
 }
@@ -37,6 +39,10 @@ variable "allow_worker_internet_access" {
   type = bool
 }
 
+variable "allow_pod_internet_access" {
+  type = bool
+}
+
 variable "allow_worker_ssh_access" {
   type = bool
 }