using separate subnets/security lists for NAT instances

Author: srig

docs/input-variables.md

@@ -135,6 +135,7 @@ The following input variables are used to configure the inbound security rules f
 
 name                                | default                 | description
 ------------------------------------|-------------------------|------------
+dedicated_nat_subnets               | "true"                  | whether to provision dedicated subnets in each AD that are only used by NAT instance(s) (separate subnets = separate control)
 public_subnet_ssh_ingress           | 0.0.0.0/0               | A CIDR notation IP range that is allowed to SSH to instances in the public subnet (including NAT instances)
 public_subnet_http_ingress          | 0.0.0.0/0               | A CIDR notation IP range that is allowed access to port 80 on instances in the public subnet
 public_subnet_https_ingress         | 0.0.0.0/0               | A CIDR notation IP range that is allowed access to port 443 on instances in the public subnet

k8s-oci.tf

@@ -55,6 +55,7 @@ module "vcn" {
   nat_instance_ad2_enabled                = "${var.nat_instance_ad2_enabled}"
   nat_instance_ad3_enabled                = "${var.nat_instance_ad3_enabled}"
   nat_instance_ssh_public_key_openssh     = "${module.k8s-tls.ssh_public_key_openssh}"
+  dedicated_nat_subnets                   = "${var.dedicated_nat_subnets}"
   worker_ssh_ingress                      = "${var.worker_ssh_ingress}"
   worker_nodeport_ingress                 = "${var.worker_nodeport_ingress}"
   external_icmp_ingress                   = "${var.external_icmp_ingress}"

network/vcn/natinstance.tf

@@ -11,7 +11,7 @@ resource "oci_core_instance" "NATInstanceAD1" {
   shape               = "${var.nat_instance_shape}"
 
   create_vnic_details {
-    subnet_id = "${oci_core_subnet.PublicSubnetAD1.id}"
+    subnet_id = "${(var.control_plane_subnet_access == "private") && (var.dedicated_nat_subnets == "true") ? "${element(concat(oci_core_subnet.NATSubnetAD1.*.id,list("")),0)}" : "${oci_core_subnet.PublicSubnetAD1.id}"}"
 
     # Skip the source/destination check so that the VNIC will forward traffic.
     skip_source_dest_check = true
@@ -38,7 +38,7 @@ resource "oci_core_instance" "NATInstanceAD2" {
   shape               = "${var.nat_instance_shape}"
 
   create_vnic_details {
-    subnet_id = "${oci_core_subnet.PublicSubnetAD2.id}"
+    subnet_id = "${(var.control_plane_subnet_access == "private") && (var.dedicated_nat_subnets == "true") ? "${element(concat(oci_core_subnet.NATSubnetAD2.*.id,list("")),0)}" : "${oci_core_subnet.PublicSubnetAD2.id}"}"
 
     # Skip the source/destination check so that the VNIC will forward traffic.
     skip_source_dest_check = true
@@ -63,7 +63,7 @@ resource "oci_core_instance" "NATInstanceAD3" {
   shape               = "${var.nat_instance_shape}"
 
   create_vnic_details {
-    subnet_id = "${oci_core_subnet.PublicSubnetAD3.id}"
+    subnet_id = "${(var.control_plane_subnet_access == "private") && (var.dedicated_nat_subnets == "true") ? "${element(concat(oci_core_subnet.NATSubnetAD3.*.id,list("")),0)}" : "${oci_core_subnet.PublicSubnetAD3.id}"}"
 
     # Skip the source/destination check so that the VNIC will forward traffic.
     skip_source_dest_check = true

network/vcn/outputs.tf

@@ -50,6 +50,18 @@ output "public_subnet_ad3_id" {
   value = "${oci_core_subnet.PublicSubnetAD3.*.id}"
 }
 
+output "nat_subnet_ad1_id" {
+  value = "${oci_core_subnet.NATSubnetAD1.*.id}"
+}
+
+output "nat_subnet_ad2_id" {
+  value = "${oci_core_subnet.NATSubnetAD2.*.id}"
+}
+
+output "nat_subnet_ad3_id" {
+  value = "${oci_core_subnet.NATSubnetAD3.*.id}"
+}
+
 output "ccmlb_subnet_ad1_id" {
   value = "${oci_core_subnet.k8sCCMLBSubnetAD1.*.id}"
 }

network/vcn/securitylists.tf

@@ -300,6 +300,91 @@ resource "oci_core_security_list" "PublicSecurityList" {
   ]
 }
 
+resource "oci_core_security_list" "NatSecurityList" {
+  count          = "${(var.control_plane_subnet_access == "private") && (var.dedicated_nat_subnets == "true") ? "1" : "0"}"
+  compartment_id = "${var.compartment_ocid}"
+  display_name   = "nat_security_list"
+  vcn_id         = "${oci_core_virtual_network.CompleteVCN.id}"
+
+  egress_security_rules = [{
+    protocol    = "all"
+    destination = "0.0.0.0/0"
+  }]
+
+  ingress_security_rules = [
+    {
+      protocol = "1"
+      source   = "${var.external_icmp_ingress}"
+
+      icmp_options {
+        "type" = 3
+        "code" = 4
+      }
+    },
+    {
+      protocol = "1"
+      source   = "${var.internal_icmp_ingress}"
+
+      icmp_options {
+        "type" = 3
+        "code" = 4
+      }
+    },
+    {
+      # Allow LBaaS
+      protocol = "6"
+      source   = "${lookup(var.bmc_ingress_cidrs, "LBAAS-PHOENIX-1-CIDR")}"
+    },
+    {
+      protocol = "6"
+      source   = "${lookup(var.bmc_ingress_cidrs, "LBAAS-ASHBURN-1-CIDR")}"
+    },
+    {
+      # Allow internal VCN traffic
+      protocol = "all"
+      source   = "${lookup(var.bmc_ingress_cidrs, "VCN-CIDR")}"
+    },
+    {
+      # Access to SSH port to instances on the public network (like the NAT instance or a user-defined LB)
+      protocol = "6"
+      source   = "${var.public_subnet_ssh_ingress}"
+
+      tcp_options {
+        "min" = 22
+        "max" = 22
+      }
+    },
+    {
+      # Access to port 80 and 443 to instances on the public network (like the NAT instance or a user-defined LB)
+      protocol = "6"
+      source   = "${var.public_subnet_http_ingress}"
+
+      tcp_options {
+        "min" = 80
+        "max" = 80
+      }
+    },
+    {
+      protocol = "6"
+      source   = "${var.public_subnet_https_ingress}"
+
+      tcp_options {
+        "min" = 443
+        "max" = 443
+      }
+    },
+    {
+      protocol = "6"
+      source   = "${var.etcd_cluster_ingress}"
+      
+      tcp_options {
+        "min" = 2379
+        "max" = 2380
+      }
+    },
+  ]
+}
+
 resource "oci_core_security_list" "K8SCCMLBSubnet" {
   compartment_id = "${var.compartment_ocid}"
   display_name   = "${var.label_prefix}k8sCCM_security_list"

network/vcn/subnets.tf

@@ -37,6 +37,43 @@ resource "oci_core_subnet" "PublicSubnetAD3" {
   dhcp_options_id     = "${oci_core_virtual_network.CompleteVCN.default_dhcp_options_id}"
 }
 
+resource "oci_core_subnet" "NATSubnetAD1" {
+  # Provisioned only when k8s instances are in private subnets
+  count               = "${(var.control_plane_subnet_access == "private") && (var.dedicated_nat_subnets == "true") ? "1" : "0"}"
+  availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[0],"name")}"
+  cidr_block          = "${lookup(var.network_cidrs, "natSubnetAD1")}"
+  display_name        = "${var.label_prefix}publicNATSubnetAD1"
+  compartment_id      = "${var.compartment_ocid}"
+  vcn_id              = "${oci_core_virtual_network.CompleteVCN.id}"
+  route_table_id      = "${oci_core_route_table.PublicRouteTable.id}"
+  security_list_ids   = ["${concat(list(oci_core_security_list.NatSecurityList.id), var.additional_nat_security_lists_ids)}"]
+  dhcp_options_id     = "${oci_core_virtual_network.CompleteVCN.default_dhcp_options_id}"
+}
+
+resource "oci_core_subnet" "NATSubnetAD2" {
+  count               = "${(var.control_plane_subnet_access == "private") && (var.dedicated_nat_subnets == "true") ? "1" : "0"}"
+  availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[1],"name")}"
+  cidr_block          = "${lookup(var.network_cidrs, "natSubnetAD2")}"
+  display_name        = "${var.label_prefix}publicNATSubnetAD2"
+  compartment_id      = "${var.compartment_ocid}"
+  vcn_id              = "${oci_core_virtual_network.CompleteVCN.id}"
+  route_table_id      = "${oci_core_route_table.PublicRouteTable.id}"
+  security_list_ids   = ["${concat(list(oci_core_security_list.NatSecurityList.id), var.additional_nat_security_lists_ids)}"]
+  dhcp_options_id     = "${oci_core_virtual_network.CompleteVCN.default_dhcp_options_id}"
+}
+
+resource "oci_core_subnet" "NATSubnetAD3" {
+  count               = "${(var.control_plane_subnet_access == "private") && (var.dedicated_nat_subnets == "true") ? "1" : "0"}"
+  availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[2],"name")}"
+  cidr_block          = "${lookup(var.network_cidrs, "natSubnetAD3")}"
+  display_name        = "${var.label_prefix}publicNATSubnetAD3"
+  compartment_id      = "${var.compartment_ocid}"
+  vcn_id              = "${oci_core_virtual_network.CompleteVCN.id}"
+  route_table_id      = "${oci_core_route_table.PublicRouteTable.id}"
+  security_list_ids   = ["${concat(list(oci_core_security_list.NatSecurityList.id), var.additional_nat_security_lists_ids)}"]
+  dhcp_options_id     = "${oci_core_virtual_network.CompleteVCN.default_dhcp_options_id}"
+}
+
 resource "oci_core_subnet" "etcdSubnetAD1" {
   availability_domain = "${lookup(data.oci_identity_availability_domains.ADs.availability_domains[0],"name")}"
   cidr_block          = "${lookup(var.network_cidrs, "etcdSubnetAD1")}"

network/vcn/variables.tf

@@ -6,6 +6,9 @@ variable "network_cidrs" {
     PublicSubnetAD1   = "10.0.10.0/24"
     PublicSubnetAD2   = "10.0.11.0/24"
     PublicSubnetAD3   = "10.0.12.0/24"
+    natSubnetAD1      = "10.0.13.0/24"
+    natSubnetAD2      = "10.0.14.0/24"
+    natSubnetAD3      = "10.0.15.0/24"
     etcdSubnetAD1     = "10.0.20.0/24"
     etcdSubnetAD2     = "10.0.21.0/24"
     etcdSubnetAD3     = "10.0.22.0/24"
@@ -47,6 +50,11 @@ variable "additional_public_security_lists_ids" {
   default = []
 }
 
+variable "additional_nat_security_lists_ids" {
+  type    = "list"
+  default = []
+}
+
 # VCN
 
 variable "label_prefix" {
@@ -136,3 +144,7 @@ variable nat_instance_ad2_enabled {
 variable nat_instance_ad3_enabled {
   default = "false"
 }
+
+variable dedicated_nat_subnets {
+  default = "false"
+}

outputs.tf

@@ -54,6 +54,10 @@ output "public_subnet_ids" {
   value = ["${module.vcn.public_subnet_ad1_id}", "${module.vcn.public_subnet_ad2_id}", "${module.vcn.public_subnet_ad3_id}", ""]
 }
 
+output "nat_subnet_ids" {
+  value = ["${module.vcn.nat_subnet_ad1_id}", "${module.vcn.nat_subnet_ad2_id}", "${module.vcn.nat_subnet_ad3_id}", ""]
+}
+
 output "worker_ssh_ingress_cidr" {
   value = "${var.worker_ssh_ingress}"
 }

tests/resources/frontend-service.yml

@@ -31,4 +31,4 @@ spec:
           lifecycle:
             preStop:
               exec:
-                command: ["/usr/sbin/nginx","-s","quit"]
+                command: ["/usr/sbin/nginx","-s","quit"]

tests/resources/hello-service.yml

@@ -30,4 +30,4 @@ spec:
           image: "gcr.io/google-samples/hello-go-gke:1.0"
           ports:
             - name: http
-              containerPort: 80
+              containerPort: 80