From 0353b399ccfe72c26e8d3ca828a9c7f56927cb3a Mon Sep 17 00:00:00 2001 From: Ali Mukadam Date: Mon, 8 Aug 2022 11:21:52 +1000 Subject: [PATCH 1/4] feat: added new subnet for control plane endpoint, renamed other subnets accordingly Signed-off-by: Ali Mukadam --- modules/network/locals.tf | 14 ++++++++------ modules/network/outputs.tf | 9 +++++---- modules/network/seclist.tf | 4 ++-- modules/network/subnets.tf | 27 +++++++++++++++++++-------- terraform.tfvars.example | 14 +++++++------- variables.tf | 21 +++++++++++---------- 6 files changed, 52 insertions(+), 37 deletions(-) diff --git a/modules/network/locals.tf b/modules/network/locals.tf index d470e7f..c4b7221 100644 --- a/modules/network/locals.tf +++ b/modules/network/locals.tf @@ -8,17 +8,19 @@ locals { vcn_cidr = element(data.oci_core_vcn.vcn.cidr_blocks, 0) # subnet cidrs - used by subnets - bastion_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["bastion"], "newbits"), lookup(var.subnets["bastion"], "netnum")) + bastion-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["bastion"], "newbits"), lookup(var.subnets["bastion"], "netnum")) - cp_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["cp"], "newbits"), lookup(var.subnets["cp"], "netnum")) + operator-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["operator"], "newbits"), lookup(var.subnets["operator"], "netnum")) - int_lb_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["int_lb"], "newbits"), lookup(var.subnets["int_lb"], "netnum")) + cp-endpoint-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["cp-endpoint"], "newbits"), lookup(var.subnets["cp-endpoint"], "netnum")) - operator_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["operator"], "newbits"), lookup(var.subnets["operator"], "netnum")) + cp-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["cp"], "newbits"), lookup(var.subnets["cp"], "netnum")) - pub_lb_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["pub_lb"], "newbits"), lookup(var.subnets["pub_lb"], "netnum")) + service-lb-int-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["service-lb-int"], "newbits"), lookup(var.subnets["service-lb-int"], "netnum")) - workers_subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["workers"], "newbits"), lookup(var.subnets["workers"], "netnum")) + service-lb-pub-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["service-lb-pub"], "newbits"), lookup(var.subnets["service-lb-pub"], "netnum")) + + workers-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["workers"], "newbits"), lookup(var.subnets["workers"], "netnum")) anywhere = "0.0.0.0/0" diff --git a/modules/network/outputs.tf b/modules/network/outputs.tf index e96d6b0..8ab70db 100644 --- a/modules/network/outputs.tf +++ b/modules/network/outputs.tf @@ -3,9 +3,10 @@ output "subnet_ids" { value = { - "cp" = join(",", oci_core_subnet.cp[*].id) - "workers" = join(",", oci_core_subnet.workers[*].id) - "int_lb" = join(",", oci_core_subnet.int_lb[*].id) - "pub_lb" = join(",", oci_core_subnet.pub_lb[*].id) + "cp" = join(",", oci_core_subnet.cp[*].id) + "cp-endpoint" = join(",", oci_core_subnet.cp-endpoint[*].id) + "workers" = join(",", oci_core_subnet.workers[*].id) + "service-lb-int-subnet" = join(",", oci_core_subnet.service-lb-int-subnet[*].id) + "service-lb-int-subnet" = join(",", oci_core_subnet.service-lb-pub-subnet[*].id) } } diff --git a/modules/network/seclist.tf b/modules/network/seclist.tf index 893e7ba..4e557e9 100644 --- a/modules/network/seclist.tf +++ b/modules/network/seclist.tf @@ -8,7 +8,7 @@ resource "oci_core_security_list" "control_plane_seclist" { egress_security_rules { description = "Allow Bastion service to communicate to the control plane endpoint. Required for when using OCI Bastion service." - destination = local.cp_subnet + destination = local.cp-subnet destination_type = "CIDR_BLOCK" protocol = local.tcp_protocol stateless = false @@ -22,7 +22,7 @@ resource "oci_core_security_list" "control_plane_seclist" { ingress_security_rules { description = "Allow Bastion service to communicate to the control plane endpoint. Required for when using OCI Bastion service." protocol = local.tcp_protocol - source = local.cp_subnet + source = local.cp-endpoint-subnet source_type = "CIDR_BLOCK" stateless = false diff --git a/modules/network/subnets.tf b/modules/network/subnets.tf index 2002d33..d46945e 100644 --- a/modules/network/subnets.tf +++ b/modules/network/subnets.tf @@ -2,7 +2,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl resource "oci_core_subnet" "cp" { - cidr_block = local.cp_subnet + cidr_block = local.cp-subnet compartment_id = var.compartment_id display_name = var.label_prefix == "none" ? "control-plane" : "${var.label_prefix}-control-plane" dns_label = "cp" @@ -12,8 +12,19 @@ resource "oci_core_subnet" "cp" { vcn_id = var.vcn_id } +resource "oci_core_subnet" "cp-endpoint" { + cidr_block = local.cp-endpoint-subnet + compartment_id = var.compartment_id + display_name = var.label_prefix == "none" ? "control-plane-endpoint" : "${var.label_prefix}-control-plane-endpoint" + dns_label = "cp-endpoint" + prohibit_public_ip_on_vnic = var.control_plane_type == "private" ? true : false + route_table_id = var.control_plane_type == "private" ? var.nat_route_id : var.ig_route_id + security_list_ids = [oci_core_security_list.control_plane_seclist.id] + vcn_id = var.vcn_id +} + resource "oci_core_subnet" "workers" { - cidr_block = local.workers_subnet + cidr_block = local.workers-subnet compartment_id = var.compartment_id display_name = var.label_prefix == "none" ? "workers" : "${var.label_prefix}-workers" dns_label = "workers" @@ -22,10 +33,10 @@ resource "oci_core_subnet" "workers" { vcn_id = var.vcn_id } -resource "oci_core_subnet" "int_lb" { - cidr_block = local.int_lb_subnet +resource "oci_core_subnet" "service-lb-int-subnet" { + cidr_block = local.service-lb-int-subnet compartment_id = var.compartment_id - display_name = var.label_prefix == "none" ? "int-lb" : "${var.label_prefix}-int-lb" + display_name = var.label_prefix == "none" ? "svc-lb-int" : "${var.label_prefix}-svc-lb-int" dns_label = "intlb" prohibit_public_ip_on_vnic = true route_table_id = var.nat_route_id @@ -34,10 +45,10 @@ resource "oci_core_subnet" "int_lb" { count = var.load_balancers == "internal" || var.load_balancers == "both" ? 1 : 0 } -resource "oci_core_subnet" "pub_lb" { - cidr_block = local.pub_lb_subnet +resource "oci_core_subnet" "service-lb-pub-subnet" { + cidr_block = local.service-lb-pub-subnet compartment_id = var.compartment_id - display_name = var.label_prefix == "none" ? "pub-lb" : "${var.label_prefix}-pub-lb" + display_name = var.label_prefix == "none" ? "svc-lb-pub" : "${var.label_prefix}-svc-lb-pub" dns_label = "publb" prohibit_public_ip_on_vnic = false route_table_id = var.ig_route_id diff --git a/terraform.tfvars.example b/terraform.tfvars.example index d6a9daf..ab357d4 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -63,13 +63,13 @@ nat_gateway_route_rules = [ nat_gateway_public_ip_id = "none" subnets = { - bastion = { netnum = 0, newbits = 13 } - operator = { netnum = 1, newbits = 13 } - cp = { netnum = 2, newbits = 13 } - int_lb = { netnum = 16, newbits = 11 } - pub_lb = { netnum = 17, newbits = 11 } - workers = { netnum = 1, newbits = 2 } - fss = { netnum = 18, newbits = 11 } + bastion = { netnum = 0, newbits = 14 } + operator = { netnum = 1, newbits = 14 } + cp-endpoint = { netnum = 1, newbits = 13 } + cp = { netnum = 2, newbits = 13 } + service-lb-int = { netnum = 1, newbits = 11 } + service-lb-pub = { netnum = 2, newbits = 11 } + workers = { netnum = 1, newbits = 6 } } create_vcn = true diff --git a/variables.tf b/variables.tf index f481f98..3929f5d 100644 --- a/variables.tf +++ b/variables.tf @@ -147,10 +147,10 @@ variable "drg_display_name" { default = "drg" } -variable "drg_id"{ +variable "drg_id" { description = "ID of an external created Dynamic Routing Gateway to be attached to the VCN" - type = string - default = null + type = string + default = null } variable "internet_gateway_route_rules" { @@ -186,12 +186,13 @@ variable "nat_gateway_public_ip_id" { variable "subnets" { description = "parameters to cidrsubnet function to calculate subnet masks within the VCN." default = { - bastion = { netnum = 0, newbits = 13 } - operator = { netnum = 1, newbits = 13 } - cp = { netnum = 2, newbits = 13 } - int_lb = { netnum = 16, newbits = 11 } - pub_lb = { netnum = 17, newbits = 11 } - workers = { netnum = 1, newbits = 2 } + bastion = { netnum = 0, newbits = 14 } + operator = { netnum = 1, newbits = 14 } + cp-endpoint = { netnum = 1, newbits = 13 } + cp = { netnum = 2, newbits = 13 } + service-lb-int = { netnum = 1, newbits = 11 } + service-lb-pub = { netnum = 2, newbits = 11 } + workers = { netnum = 1, newbits = 6 } } type = map(any) } @@ -260,6 +261,6 @@ variable "freeform_tags" { } description = "Tags to apply to different resources." type = object({ - vcn = map(any), + vcn = map(any), }) } From 6528ea38b3173cf48c4eb66ca4d466d64bfe06fd Mon Sep 17 00:00:00 2001 From: Ali Mukadam Date: Mon, 8 Aug 2022 13:00:39 +1000 Subject: [PATCH 2/4] fix: changed dns label for control plane endpoint, output for subnet ids Signed-off-by: Ali Mukadam --- modules/network/outputs.tf | 2 +- modules/network/subnets.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/network/outputs.tf b/modules/network/outputs.tf index 8ab70db..b7d9516 100644 --- a/modules/network/outputs.tf +++ b/modules/network/outputs.tf @@ -7,6 +7,6 @@ output "subnet_ids" { "cp-endpoint" = join(",", oci_core_subnet.cp-endpoint[*].id) "workers" = join(",", oci_core_subnet.workers[*].id) "service-lb-int-subnet" = join(",", oci_core_subnet.service-lb-int-subnet[*].id) - "service-lb-int-subnet" = join(",", oci_core_subnet.service-lb-pub-subnet[*].id) + "service-lb-pub-subnet" = join(",", oci_core_subnet.service-lb-pub-subnet[*].id) } } diff --git a/modules/network/subnets.tf b/modules/network/subnets.tf index d46945e..0b7735e 100644 --- a/modules/network/subnets.tf +++ b/modules/network/subnets.tf @@ -16,7 +16,7 @@ resource "oci_core_subnet" "cp-endpoint" { cidr_block = local.cp-endpoint-subnet compartment_id = var.compartment_id display_name = var.label_prefix == "none" ? "control-plane-endpoint" : "${var.label_prefix}-control-plane-endpoint" - dns_label = "cp-endpoint" + dns_label = "cpendpoint" prohibit_public_ip_on_vnic = var.control_plane_type == "private" ? true : false route_table_id = var.control_plane_type == "private" ? var.nat_route_id : var.ig_route_id security_list_ids = [oci_core_security_list.control_plane_seclist.id] From 191c083d05dadf01c3b1ae49037a7278afe61155 Mon Sep 17 00:00:00 2001 From: Ali Mukadam Date: Tue, 9 Aug 2022 11:13:07 +1000 Subject: [PATCH 3/4] feat: added nsg for control plane nodes for antrea Signed-off-by: Ali Mukadam --- main.tf | 37 +- modules/antrea/datasources.tf | 24 ++ modules/antrea/locals.tf | 268 ++++++++++++++ modules/antrea/nsgs.tf | 636 ++++++++++++++++++++++++++++++++++ modules/antrea/variables.tf | 58 ++++ modules/antrea/versions.tf | 13 + modules/network/seclist.tf | 12 +- modules/network/subnets.tf | 3 +- 8 files changed, 1039 insertions(+), 12 deletions(-) create mode 100644 modules/antrea/datasources.tf create mode 100644 modules/antrea/locals.tf create mode 100644 modules/antrea/nsgs.tf create mode 100644 modules/antrea/variables.tf create mode 100644 modules/antrea/versions.tf diff --git a/main.tf b/main.tf index a6fb6f1..b78c1cc 100644 --- a/main.tf +++ b/main.tf @@ -57,7 +57,7 @@ module "drg" { count = var.create_drg || var.drg_id != null ? 1 : 0 } -# additional networking for oke +# additional networking for subnets module "network" { source = "./modules/network" @@ -65,7 +65,7 @@ module "network" { compartment_id = var.compartment_id label_prefix = var.label_prefix - # oke networking parameters + # networking parameters ig_route_id = local.ig_route_id nat_route_id = local.nat_route_id subnets = var.subnets @@ -73,10 +73,10 @@ module "network" { # control plane endpoint parameters - control_plane_type = var.control_plane_type + control_plane_type = var.control_plane_type # worker network parameters - worker_type = var.worker_type + worker_type = var.worker_type # oke load balancer network parameters load_balancers = var.load_balancers @@ -85,3 +85,32 @@ module "network" { module.vcn ] } + +# nsgs for antrea cni +module "antrea" { + source = "./modules/antrea" + + # general oci parameters + compartment_id = var.compartment_id + label_prefix = var.label_prefix + + # networking parameters + subnets = var.subnets + vcn_id = local.vcn_id + + # control plane endpoint parameters + control_plane_type = "public" + control_plane_allowed_cidrs = ["0.0.0.0/0"] + + # worker network parameters + allow_node_port_access = false + allow_worker_internet_access = true + worker_type = var.worker_type + + # load balancer network parameters + # load_balancers = var.load_balancers + + depends_on = [ + module.network + ] +} diff --git a/modules/antrea/datasources.tf b/modules/antrea/datasources.tf new file mode 100644 index 0000000..6ebc9a1 --- /dev/null +++ b/modules/antrea/datasources.tf @@ -0,0 +1,24 @@ +# Copyright (c) 2022 Oracle Corporation and/or affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +data "oci_core_services" "all_oci_services" { + filter { + name = "name" + values = ["All .* Services In Oracle Services Network"] + regex = true + } +} + +data "oci_core_subnets" "subnets" { + compartment_id = var.compartment_id + vcn_id = var.vcn_id + + filter { + name = "state" + values = ["AVAILABLE"] + } +} + +data "oci_core_vcn" "vcn" { + vcn_id = var.vcn_id +} diff --git a/modules/antrea/locals.tf b/modules/antrea/locals.tf new file mode 100644 index 0000000..4a59e0b --- /dev/null +++ b/modules/antrea/locals.tf @@ -0,0 +1,268 @@ +# Copyright (c) 2022 Oracle Corporation and/or affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +locals { + + # first vcn cidr + # pick the first cidr block in the list as this is where we will create the oke subnets + vcn_cidr = element(data.oci_core_vcn.vcn.cidr_blocks, 0) + + # subnet cidrs - used by subnets + bastion-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["bastion"], "newbits"), lookup(var.subnets["bastion"], "netnum")) + + operator-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["operator"], "newbits"), lookup(var.subnets["operator"], "netnum")) + + cp-endpoint-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["cp-endpoint"], "newbits"), lookup(var.subnets["cp-endpoint"], "netnum")) + + cp-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["cp"], "newbits"), lookup(var.subnets["cp"], "netnum")) + + service-lb-int-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["service-lb-int"], "newbits"), lookup(var.subnets["service-lb-int"], "netnum")) + + service-lb-pub-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["service-lb-pub"], "newbits"), lookup(var.subnets["service-lb-pub"], "netnum")) + + workers-subnet = cidrsubnet(local.vcn_cidr, lookup(var.subnets["workers"], "newbits"), lookup(var.subnets["workers"], "netnum")) + + anywhere = "0.0.0.0/0" + + # port numbers + ssh_port = 22 + + # protocols + # # special OCI value for all protocols + all_protocols = "all" + + # # IANA protocol numbers + icmp_protocol = 1 + + tcp_protocol = 6 + + udp_protocol = 17 + + # oracle services network + osn = lookup(data.oci_core_services.all_oci_services.services[0], "cidr_block") + + # port numbers + health_check_port = 10256 + node_port_min = 30000 + node_port_max = 32767 + + # if port = -1, allow all ports + + # control plane + cp_egress = [ + { + description = "Allow Kubernetes control plane to anywhere", + destination = local.anywhere, + destination_type = "CIDR_BLOCK", + protocol = local.all_protocols, + port = -1, + stateless = false + }, + { + description = "Allow control nodes to communicate with OCI services", + destination = local.osn, + destination_type = "SERVICE_CIDR_BLOCK", + protocol = local.tcp_protocol, + port = -1, + stateless = false + } + ] + + cp_ingress = [ + { + description = "Allow control plane API endpoint to control plane nodes" + protocol = local.tcp_protocol, + port = 6443, + source = local.cp-endpoint-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow control plane to control plane nodes (api server port)" + protocol = local.tcp_protocol, + port = 6443, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow etcd client communication" + protocol = local.tcp_protocol, + port = 2379, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Antrea service" + protocol = local.tcp_protocol, + port = 10349, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Geneve service" + protocol = local.udp_protocol, + port = 6081, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Geneve service" + protocol = local.udp_protocol, + port = 6081, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Path discovery" + protocol = local.icmp_protocol, + port = -1, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + ] + + # # workers + # workers_egress = [ + # { + # description = "Allow ICMP traffic for path discovery", + # destination = local.anywhere + # destination_type = "CIDR_BLOCK", + # protocol = local.icmp_protocol, + # port = -1, + # stateless = false + # }, + # { + # description = "Allow worker nodes to communicate with OKE", + # destination = local.osn, + # destination_type = "SERVICE_CIDR_BLOCK", + # protocol = local.tcp_protocol, + # port = -1, + # stateless = false + # }, + # { + # description = "Allow worker nodes to control plane API endpoint communication", + # destination = local.cp_subnet, + # destination_type = "CIDR_BLOCK", + # protocol = local.tcp_protocol, + # port = 6443, + # stateless = false + # }, + # { + # description = "Allow worker nodes to control plane communication", + # destination = local.cp_subnet, + # destination_type = "CIDR_BLOCK", + # protocol = local.tcp_protocol, + # port = 12250, + # stateless = false + # } + # ] + + # workers_ingress = [ + # { + # description = "Allow ingress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet", + # protocol = local.all_protocols, + # port = -1, + # source = local.workers_subnet, + # source_type = "CIDR_BLOCK", + # stateless = false + # }, + # { + # description = "Allow control plane to communicate with worker nodes", + # protocol = local.tcp_protocol, + # port = 10250, + # source = local.cp_subnet, + # source_type = "CIDR_BLOCK", + # stateless = false + # }, + # { + # description = "Allow path discovery from worker nodes" + # protocol = local.icmp_protocol, + # port = -1, + # //this should be local.worker_subnet? + # source = local.anywhere, + # source_type = "CIDR_BLOCK", + # stateless = false + # } + # ] + + # int_lb_egress = [ + # { + # description = "Allow stateful egress to workers. Required for NodePorts", + # destination = local.workers_subnet, + # destination_type = "CIDR_BLOCK", + # protocol = local.tcp_protocol, + # port = "30000-32767", + # stateless = false + # }, + # { + # description = "Allow ICMP traffic for path discovery to worker nodes", + # destination = local.workers_subnet, + # destination_type = "CIDR_BLOCK", + # protocol = local.icmp_protocol, + # port = -1, + # stateless = false + # }, + # { + # description = "Allow stateful egress to workers. Required for load balancer http/tcp health checks", + # destination = local.workers_subnet, + # destination_type = "CIDR_BLOCK", + # protocol = local.tcp_protocol, + # port = local.health_check_port, + # stateless = false + # }, + # ] + + # # Combine supplied allow list and the public load balancer subnet + # internal_lb_allowed_cidrs = var.load_balancers == "both" ? concat(var.internal_lb_allowed_cidrs, tolist([local.pub_lb_subnet])) : var.internal_lb_allowed_cidrs + + # # Create a Cartesian product of allowed cidrs and ports + # internal_lb_allowed_cidrs_and_ports = setproduct(local.internal_lb_allowed_cidrs, var.internal_lb_allowed_ports) + + # pub_lb_egress = [ + # # { + # # description = "Allow stateful egress to internal load balancers subnet on port 80", + # # destination = local.int_lb_subnet, + # # destination_type = "CIDR_BLOCK", + # # protocol = local.tcp_protocol, + # # port = 80 + # # stateless = false + # # }, + # # { + # # description = "Allow stateful egress to internal load balancers subnet on port 443", + # # destination = local.int_lb_subnet, + # # destination_type = "CIDR_BLOCK", + # # protocol = local.tcp_protocol, + # # port = 443 + # # stateless = false + # # }, + # { + # description = "Allow stateful egress to workers. Required for NodePorts", + # destination = local.workers_subnet, + # destination_type = "CIDR_BLOCK", + # protocol = local.tcp_protocol, + # port = "30000-32767", + # stateless = false + # }, + # { + # description = "Allow ICMP traffic for path discovery to worker nodes", + # destination = local.workers_subnet, + # destination_type = "CIDR_BLOCK", + # protocol = local.icmp_protocol, + # port = -1, + # stateless = false + # }, + # ] + + # public_lb_allowed_cidrs = var.public_lb_allowed_cidrs + # public_lb_allowed_cidrs_and_ports = setproduct(local.public_lb_allowed_cidrs, var.public_lb_allowed_ports) + + + + +} diff --git a/modules/antrea/nsgs.tf b/modules/antrea/nsgs.tf new file mode 100644 index 0000000..5b7bf86 --- /dev/null +++ b/modules/antrea/nsgs.tf @@ -0,0 +1,636 @@ +# Copyright (c) 2022 Oracle Corporation and/or affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +# # control plane nsg and rules +# resource "oci_core_network_security_group" "cp" { +# compartment_id = var.compartment_id +# display_name = var.label_prefix == "none" ? "control-plane" : "${var.label_prefix}-control-plane" +# vcn_id = var.vcn_id +# } + +# resource "oci_core_network_security_group_security_rule" "cp_egress" { +# network_security_group_id = oci_core_network_security_group.cp.id +# description = local.cp_egress[count.index].description +# destination = local.cp_egress[count.index].destination +# destination_type = local.cp_egress[count.index].destination_type +# direction = "EGRESS" +# protocol = local.cp_egress[count.index].protocol + +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.cp_egress[count.index].protocol == local.tcp_protocol && local.cp_egress[count.index].port != -1 ? [1] : [] +# content { +# destination_port_range { +# min = local.cp_egress[count.index].port +# max = local.cp_egress[count.index].port +# } +# } +# } + +# dynamic "icmp_options" { +# for_each = local.cp_egress[count.index].protocol == local.icmp_protocol ? [1] : [] +# content { +# type = 3 +# code = 4 +# } +# } + +# count = length(local.cp_egress) +# } + +# resource "oci_core_network_security_group_security_rule" "cp_egress_npn" { +# network_security_group_id = oci_core_network_security_group.cp.id +# description = "Allow Kubernetes Control plane to communicate with pods" +# destination = local.pods_subnet +# destination_type = "CIDR_BLOCK" +# direction = "EGRESS" +# protocol = local.all_protocols + +# stateless = false + +# count = var.cni_type == "npn" ? 1 :0 + +# } + +# resource "oci_core_network_security_group_security_rule" "cp_ingress" { +# network_security_group_id = oci_core_network_security_group.cp.id +# description = local.cp_ingress[count.index].description +# direction = "INGRESS" +# protocol = local.cp_ingress[count.index].protocol +# source = local.cp_ingress[count.index].source +# source_type = local.cp_ingress[count.index].source_type + +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.cp_ingress[count.index].protocol == local.tcp_protocol ? [1] : [] +# content { +# destination_port_range { +# min = local.cp_ingress[count.index].port +# max = local.cp_ingress[count.index].port +# } +# } +# } + +# dynamic "icmp_options" { +# for_each = local.cp_ingress[count.index].protocol == local.icmp_protocol ? [1] : [] +# content { +# type = 3 +# code = 4 +# } +# } + +# count = length(local.cp_ingress) + +# } + +# resource "oci_core_network_security_group_security_rule" "cp_ingress_additional_cidrs" { +# network_security_group_id = oci_core_network_security_group.cp.id +# description = "Allow additional CIDR block access to control plane. Required for kubectl/helm." +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = element(var.control_plane_allowed_cidrs, count.index) +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = 6443 +# max = 6443 +# } +# } + +# icmp_options { +# type = 3 +# code = 4 +# } + +# count = length(var.control_plane_allowed_cidrs) + +# } + +# # workers nsg and rules +# resource "oci_core_network_security_group" "workers" { +# compartment_id = var.compartment_id +# display_name = var.label_prefix == "none" ? "workers" : "${var.label_prefix}-workers" +# vcn_id = var.vcn_id +# } + +# resource "oci_core_network_security_group_security_rule" "workers_egress" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = local.workers_egress[count.index].description +# destination = local.workers_egress[count.index].destination +# destination_type = local.workers_egress[count.index].destination_type +# direction = "EGRESS" +# protocol = local.workers_egress[count.index].protocol + +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.workers_egress[count.index].protocol == local.tcp_protocol && local.workers_egress[count.index].port != -1 ? [1] : [] +# content { +# destination_port_range { +# min = local.workers_egress[count.index].port +# max = local.workers_egress[count.index].port +# } +# } +# } + +# dynamic "icmp_options" { +# for_each = local.workers_egress[count.index].protocol == local.icmp_protocol ? [1] : [] +# content { +# type = 3 +# code = 4 +# } +# } + +# count = length(local.workers_egress) +# } + +# resource "oci_core_network_security_group_security_rule" "workers_egress_flannel" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow egress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet" +# destination = local.workers_subnet +# destination_type = "CIDR_BLOCK" +# direction = "EGRESS" +# protocol = local.all_protocols + +# stateless = false + +# count = var.cni_type == "flannel" ? 1: 0 +# } + +# resource "oci_core_network_security_group_security_rule" "workers_egress_npn" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow worker nodes access to pods" +# destination = local.pods_subnet +# destination_type = "CIDR_BLOCK" +# direction = "EGRESS" +# protocol = local.all_protocols + +# stateless = false + +# count = var.cni_type == "npn" ? 1: 0 +# } + +# # add this rule separately so it can be controlled independently +# resource "oci_core_network_security_group_security_rule" "workers_egress_internet" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow worker nodes access to Internet. Required for getting container images or using external services" +# destination = local.anywhere +# destination_type = "CIDR_BLOCK" +# direction = "EGRESS" +# protocol = local.tcp_protocol + +# stateless = false + +# count = var.allow_worker_internet_access == true ? 1 : 0 + +# } + +# resource "oci_core_network_security_group_security_rule" "workers_ingress" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = local.workers_ingress[count.index].description +# direction = "INGRESS" +# protocol = local.workers_ingress[count.index].protocol +# source = local.workers_ingress[count.index].source +# source_type = local.workers_ingress[count.index].source_type + +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.workers_ingress[count.index].protocol == local.tcp_protocol && local.workers_ingress[count.index].port != -1 ? [1] : [] +# content { +# destination_port_range { +# min = local.workers_ingress[count.index].port +# max = local.workers_ingress[count.index].port +# } +# } +# } + +# dynamic "icmp_options" { +# for_each = local.workers_ingress[count.index].protocol == local.icmp_protocol ? [1] : [] +# content { +# type = 3 +# code = 4 +# } +# } + +# count = length(local.workers_ingress) + +# } + +# # add the next 4 rules separately so it can be controlled independently based on which lbs are created +# resource "oci_core_network_security_group_security_rule" "workers_ingress_from_int_lb" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow internal load balancers traffic to workers" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = local.int_lb_subnet +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = local.node_port_min +# max = local.node_port_max +# } +# } + +# count = var.load_balancers == "internal" || var.load_balancers == "both" ? 1 : 0 + +# } + +# resource "oci_core_network_security_group_security_rule" "workers_healthcheck_ingress_from_int_lb" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow internal load balancers health check to workers" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = local.int_lb_subnet +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = local.health_check_port +# max = local.health_check_port +# } +# } + +# count = var.load_balancers == "internal" || var.load_balancers == "both" ? 1 : 0 + +# } + +# resource "oci_core_network_security_group_security_rule" "workers_ingress_from_pub_lb" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow public load balancers traffic to workers" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = local.pub_lb_subnet +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = local.node_port_min +# max = local.node_port_max +# } +# } + +# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 + +# } + +# resource "oci_core_network_security_group_security_rule" "workers_healthcheck_ingress_from_pub_lb" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow public load balancers health check to workers" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = local.pub_lb_subnet +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = local.health_check_port +# max = local.health_check_port +# } +# } + +# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 + +# } + +# resource "oci_core_network_security_group_security_rule" "workers_ssh_ingress_from_bastion" { +# network_security_group_id = oci_core_network_security_group.workers.id +# description = "Allow ssh access to workers via Bastion host" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = local.bastion_subnet +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = local.ssh_port +# max = local.ssh_port +# } +# } + +# count = var.allow_worker_ssh_access == true ? 1 : 0 + +# } + +# # pod nsg and rules +# resource "oci_core_network_security_group" "pods" { +# compartment_id = var.compartment_id +# display_name = var.label_prefix == "none" ? "pods" : "${var.label_prefix}-pods" +# vcn_id = var.vcn_id +# } + +# resource "oci_core_network_security_group_security_rule" "pods_egress" { +# network_security_group_id = oci_core_network_security_group.pods.id +# description = local.pods_egress[count.index].description +# destination = local.pods_egress[count.index].destination +# destination_type = local.pods_egress[count.index].destination_type +# direction = "EGRESS" +# protocol = local.pods_egress[count.index].protocol + +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.pods_egress[count.index].protocol == local.tcp_protocol && local.pods_egress[count.index].port != -1 ? [1] : [] +# content { +# destination_port_range { +# min = local.pods_egress[count.index].port +# max = local.pods_egress[count.index].port +# } +# } +# } + +# dynamic "icmp_options" { +# for_each = local.pods_egress[count.index].protocol == local.icmp_protocol ? [1] : [] +# content { +# type = 3 +# code = 4 +# } +# } + +# count = var.cni_type =="npn" ? length(local.pods_egress) : 0 +# } + +# # add this rule separately so it can be controlled independently +# resource "oci_core_network_security_group_security_rule" "pods_egress_internet" { +# network_security_group_id = oci_core_network_security_group.pods.id +# description = "Allow pods access to Internet" +# destination = local.anywhere +# destination_type = "CIDR_BLOCK" +# direction = "EGRESS" +# protocol = local.tcp_protocol + +# stateless = false +# count = (var.cni_type =="npn" && var.allow_pod_internet_access == true) ? 1 : 0 + +# } + +# # internal lb nsg and rules +# resource "oci_core_network_security_group" "int_lb" { +# compartment_id = var.compartment_id +# display_name = var.label_prefix == "none" ? "int-lb" : "${var.label_prefix}-int-lb" +# vcn_id = var.vcn_id + +# count = var.load_balancers == "internal" || var.load_balancers == "both" ? 1 : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "int_lb_egress" { +# network_security_group_id = oci_core_network_security_group.int_lb[0].id +# description = local.int_lb_egress[count.index].description +# destination = local.int_lb_egress[count.index].destination +# destination_type = local.int_lb_egress[count.index].destination_type +# direction = "EGRESS" +# protocol = local.int_lb_egress[count.index].protocol + +# stateless = false +# # TODO: condition for end-to-end SSL/SSL termination +# dynamic "tcp_options" { +# for_each = local.int_lb_egress[count.index].protocol == local.tcp_protocol && local.int_lb_egress[count.index].port != -1 ? [1] : [] +# content { +# destination_port_range { +# min = length(regexall("-", local.int_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.int_lb_egress[count.index].port), 0)) : local.int_lb_egress[count.index].port +# max = length(regexall("-", local.int_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.int_lb_egress[count.index].port), 1)) : local.int_lb_egress[count.index].port +# } +# } +# } + +# dynamic "icmp_options" { +# for_each = local.int_lb_egress[count.index].protocol == local.icmp_protocol ? [1] : [] +# content { +# type = 3 +# code = 4 +# } +# } + +# count = var.load_balancers == "internal" || var.load_balancers == "both" ? length(local.int_lb_egress) : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "int_lb_ingress" { +# network_security_group_id = oci_core_network_security_group.int_lb[0].id +# description = "Allow stateful ingress from ${element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 0)} on port ${element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1)}" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 0) +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = length(regexall("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1)), 0) : element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1) +# max = length(regexall("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1)), 1) : element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1) +# } +# } + +# count = var.load_balancers == "internal" || var.load_balancers == "both" ? length(local.internal_lb_allowed_cidrs_and_ports) : 0 +# } + +# # public lb nsg and rules +# resource "oci_core_network_security_group" "pub_lb" { +# compartment_id = var.compartment_id +# display_name = var.label_prefix == "none" ? "pub-lb" : "${var.label_prefix}-pub-lb" +# vcn_id = var.vcn_id + +# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "pub_lb_egress" { +# network_security_group_id = oci_core_network_security_group.pub_lb[0].id +# description = local.pub_lb_egress[count.index].description +# destination = local.pub_lb_egress[count.index].destination +# destination_type = local.pub_lb_egress[count.index].destination_type +# direction = "EGRESS" +# protocol = local.pub_lb_egress[count.index].protocol + +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.pub_lb_egress[count.index].protocol == local.tcp_protocol && local.pub_lb_egress[count.index].port != -1 ? [1] : [] +# content { +# destination_port_range { +# min = length(regexall("-", local.pub_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.pub_lb_egress[count.index].port), 0)) : local.pub_lb_egress[count.index].port +# max = length(regexall("-", local.pub_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.pub_lb_egress[count.index].port), 1)) : local.pub_lb_egress[count.index].port +# } +# } +# } + +# dynamic "icmp_options" { +# for_each = local.pub_lb_egress[count.index].protocol == local.icmp_protocol ? [1] : [] +# content { +# type = 3 +# code = 4 +# } +# } + +# count = var.load_balancers == "public" || var.load_balancers == "both" ? length(local.pub_lb_egress) : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "pub_lb_egress_health_check_to_workers" { +# network_security_group_id = oci_core_network_security_group.pub_lb[0].id +# description = "Allow public load balancer health checks to workers" +# destination = local.workers_subnet +# destination_type = "CIDR_BLOCK" +# direction = "EGRESS" +# protocol = local.tcp_protocol + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = local.health_check_port +# max = local.health_check_port +# } +# } + +# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "pub_lb_egress_health_check_to_int_lb" { +# network_security_group_id = oci_core_network_security_group.pub_lb[0].id +# description = "Allow public load balancer health checks to internal load balancers" +# destination = local.int_lb_subnet +# destination_type = "CIDR_BLOCK" +# direction = "EGRESS" +# protocol = local.tcp_protocol + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = length(regexall("-", element(var.internal_lb_allowed_ports, count.index))) > 0 ? tonumber(element(split("-", element(var.internal_lb_allowed_ports, count.index)), 0)) : element(var.internal_lb_allowed_ports, count.index) +# max = length(regexall("-", element(var.internal_lb_allowed_ports, count.index))) > 0 ? tonumber(element(split("-", element(var.internal_lb_allowed_ports, count.index)), 1)) : element(var.internal_lb_allowed_ports, count.index) +# } +# } + +# count = var.load_balancers == "both" ? length(var.internal_lb_allowed_ports) : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "pub_lb_ingress" { +# network_security_group_id = oci_core_network_security_group.pub_lb[0].id +# description = "Allow stateful ingress from ${element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 0)} on port ${element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)}" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 0) +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = length(regexall("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)), 0) : element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1) +# max = length(regexall("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)), 1) : element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1) +# } +# } + +# count = var.load_balancers == "public" || var.load_balancers == "both" ? length(local.public_lb_allowed_cidrs_and_ports) : 0 +# } + +# # waf lb nsg and rules +# resource "oci_core_network_security_group" "waf" { +# compartment_id = var.compartment_id +# display_name = var.label_prefix == "none" ? "waf" : "${var.label_prefix}-waf" +# vcn_id = var.vcn_id + +# count = var.enable_waf == true ? 1 : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "waf_ingress" { +# for_each = var.enable_waf == true ? toset(local.waf_cidr_list) : toset([]) +# network_security_group_id = oci_core_network_security_group.waf[0].id +# description = "Allow stateful ingress from WAF" +# direction = "INGRESS" +# protocol = local.tcp_protocol +# source = each.key +# source_type = "CIDR_BLOCK" + +# stateless = false + +# tcp_options { +# destination_port_range { +# min = 443 +# max = 443 +# } +# } + +# } + +# ## fss : instance network security group rules + +# resource "oci_core_network_security_group_security_rule" "fss_inst_ingress" { +# network_security_group_id = oci_core_network_security_group.workers.id +# direction = "INGRESS" +# protocol = local.fss_inst_ingress[count.index].protocol +# source = local.fss_inst_ingress[count.index].source +# source_type = local.fss_inst_ingress[count.index].source_type +# description = local.fss_inst_ingress[count.index].description +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.fss_inst_ingress[count.index].protocol == local.tcp_protocol ? [1] : [] +# content { +# source_port_range { +# min = local.fss_inst_ingress[count.index].port +# max = local.fss_inst_ingress[count.index].port +# } +# } +# } + +# dynamic "udp_options" { +# for_each = local.fss_inst_ingress[count.index].protocol == local.udp_protocol ? [1] : [] +# content { +# source_port_range { +# min = local.fss_inst_ingress[count.index].port +# max = local.fss_inst_ingress[count.index].port +# } +# } +# } + +# count = var.create_fss ? length(local.fss_inst_ingress) : 0 +# } + +# resource "oci_core_network_security_group_security_rule" "fss_inst_egress" { +# network_security_group_id = oci_core_network_security_group.workers.id +# direction = "EGRESS" +# protocol = local.fss_inst_egress[count.index].protocol +# destination = local.fss_inst_egress[count.index].destination +# destination_type = local.fss_inst_egress[count.index].destination_type +# description = local.fss_inst_egress[count.index].description +# stateless = false + +# dynamic "tcp_options" { +# for_each = local.fss_inst_egress[count.index].protocol == local.tcp_protocol ? [1] : [] +# content { +# destination_port_range { +# min = local.fss_inst_egress[count.index].port +# max = local.fss_inst_egress[count.index].port +# } +# } +# } + +# dynamic "udp_options" { +# for_each = local.fss_inst_egress[count.index].protocol == local.udp_protocol ? [1] : [] +# content { +# destination_port_range { +# min = local.fss_inst_egress[count.index].port +# max = local.fss_inst_egress[count.index].port +# } +# } +# } + +# count = var.create_fss ? length(local.fss_inst_egress) : 0 +# } diff --git a/modules/antrea/variables.tf b/modules/antrea/variables.tf new file mode 100644 index 0000000..7c997b9 --- /dev/null +++ b/modules/antrea/variables.tf @@ -0,0 +1,58 @@ +# Copyright (c) 2022 Oracle Corporation and/or affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +# general oci parameters +variable "compartment_id" {} + +variable "label_prefix" {} + +# networking parameters +variable "subnets" { + type = map(any) +} + +variable "vcn_id" {} + +# cluster endpoint +variable "control_plane_type" { + type = string +} + +variable "control_plane_allowed_cidrs" { + type = list(string) +} + +# workers + +variable "allow_node_port_access" { + type = bool +} + +variable "allow_worker_internet_access" { + type = bool +} + +variable "worker_type" {} + +# # load balancers +# variable "load_balancers" { +# type = string +# } + +# # internal load balancers +# variable "internal_lb_allowed_cidrs" { +# type = list(any) +# } + +# variable "internal_lb_allowed_ports" { +# type = list(any) +# } + +# # public load balancers +# variable "public_lb_allowed_cidrs" { +# type = list(any) +# } + +# variable "public_lb_allowed_ports" { +# type = list(any) +# } \ No newline at end of file diff --git a/modules/antrea/versions.tf b/modules/antrea/versions.tf new file mode 100644 index 0000000..b69cd72 --- /dev/null +++ b/modules/antrea/versions.tf @@ -0,0 +1,13 @@ +# Copyright (c) 2022 Oracle Corporation and/or affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl + +terraform { + required_providers { + oci = { + source = "oracle/oci" + # pass oci home region provider explicitly for identity operations + version = ">= 4.67.3" + } + } + required_version = ">= 1.0.0" +} \ No newline at end of file diff --git a/modules/network/seclist.tf b/modules/network/seclist.tf index 4e557e9..d7fdbad 100644 --- a/modules/network/seclist.tf +++ b/modules/network/seclist.tf @@ -1,14 +1,14 @@ # Copyright (c) 2022 Oracle Corporation and/or affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl -resource "oci_core_security_list" "control_plane_seclist" { +resource "oci_core_security_list" "cp-endpoint" { compartment_id = var.compartment_id - display_name = var.label_prefix == "none" ? "control-plane" : "${var.label_prefix}-control-plane" + display_name = var.label_prefix == "none" ? "cp-endpoint" : "${var.label_prefix}-cp-endpoint" vcn_id = var.vcn_id egress_security_rules { - description = "Allow Bastion service to communicate to the control plane endpoint. Required for when using OCI Bastion service." - destination = local.cp-subnet + description = "Allow egress to anywhere." + destination = local.anywhere destination_type = "CIDR_BLOCK" protocol = local.tcp_protocol stateless = false @@ -20,9 +20,9 @@ resource "oci_core_security_list" "control_plane_seclist" { } ingress_security_rules { - description = "Allow Bastion service to communicate to the control plane endpoint. Required for when using OCI Bastion service." + description = "Allow ingress from anywhere." protocol = local.tcp_protocol - source = local.cp-endpoint-subnet + source = local.anywhere source_type = "CIDR_BLOCK" stateless = false diff --git a/modules/network/subnets.tf b/modules/network/subnets.tf index 0b7735e..403ab98 100644 --- a/modules/network/subnets.tf +++ b/modules/network/subnets.tf @@ -8,7 +8,6 @@ resource "oci_core_subnet" "cp" { dns_label = "cp" prohibit_public_ip_on_vnic = var.control_plane_type == "private" ? true : false route_table_id = var.control_plane_type == "private" ? var.nat_route_id : var.ig_route_id - security_list_ids = [oci_core_security_list.control_plane_seclist.id] vcn_id = var.vcn_id } @@ -19,7 +18,7 @@ resource "oci_core_subnet" "cp-endpoint" { dns_label = "cpendpoint" prohibit_public_ip_on_vnic = var.control_plane_type == "private" ? true : false route_table_id = var.control_plane_type == "private" ? var.nat_route_id : var.ig_route_id - security_list_ids = [oci_core_security_list.control_plane_seclist.id] + security_list_ids = [oci_core_security_list.cp-endpoint.id] vcn_id = var.vcn_id } From a9357192861bef4bcf543b5953216f2c19c2f940 Mon Sep 17 00:00:00 2001 From: Ali Mukadam Date: Tue, 6 Sep 2022 20:14:16 +1000 Subject: [PATCH 4/4] feat: added remaining nsg rules for antrea cni Signed-off-by: Ali Mukadam --- main.tf | 12 +- modules/antrea/locals.tf | 297 +++++------ modules/antrea/nsgs.tf | 956 ++++++++++++------------------------ modules/antrea/variables.tf | 35 +- variables.tf | 28 ++ 5 files changed, 532 insertions(+), 796 deletions(-) diff --git a/main.tf b/main.tf index b78c1cc..9ebdcdf 100644 --- a/main.tf +++ b/main.tf @@ -99,16 +99,20 @@ module "antrea" { vcn_id = local.vcn_id # control plane endpoint parameters - control_plane_type = "public" + control_plane_type = "public" control_plane_allowed_cidrs = ["0.0.0.0/0"] # worker network parameters - allow_node_port_access = false + allow_node_port_access = false allow_worker_internet_access = true - worker_type = var.worker_type + allow_worker_ssh_access = var.allow_worker_ssh_access + worker_type = var.worker_type # load balancer network parameters - # load_balancers = var.load_balancers + load_balancers = var.load_balancers + + public_lb_allowed_cidrs = var.public_lb_allowed_cidrs + public_lb_allowed_ports = var.public_lb_allowed_ports depends_on = [ module.network diff --git a/modules/antrea/locals.tf b/modules/antrea/locals.tf index 4a59e0b..498b159 100644 --- a/modules/antrea/locals.tf +++ b/modules/antrea/locals.tf @@ -3,7 +3,7 @@ locals { - # first vcn cidr + # first vcn cidr # pick the first cidr block in the list as this is where we will create the oke subnets vcn_cidr = element(data.oci_core_vcn.vcn.cidr_blocks, 0) @@ -85,6 +85,22 @@ locals { source_type = "CIDR_BLOCK", stateless = false }, + { + description = "Allow worker nodes to control plane nodes (api server port)" + protocol = local.tcp_protocol, + port = 6443, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow control plane to control plane kubelet communication" + protocol = local.tcp_protocol, + port = 10250, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, { description = "Allow etcd client communication" protocol = local.tcp_protocol, @@ -94,7 +110,23 @@ locals { stateless = false }, { - description = "Allow Antrea service" + description = "Allow etcd peer communication" + protocol = local.tcp_protocol, + port = 2380, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Antrea service communication from control plane" + protocol = local.tcp_protocol, + port = 10349, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Antrea service communication from workers" protocol = local.tcp_protocol, port = 10349, source = local.workers-subnet, @@ -102,7 +134,7 @@ locals { stateless = false }, { - description = "Allow Geneve service" + description = "Allow Geneve service communication from control plane" protocol = local.udp_protocol, port = 6081, source = local.cp-subnet, @@ -110,7 +142,7 @@ locals { stateless = false }, { - description = "Allow Geneve service" + description = "Allow Geneve service communication from workers" protocol = local.udp_protocol, port = 6081, source = local.workers-subnet, @@ -124,145 +156,132 @@ locals { source = local.workers-subnet, source_type = "CIDR_BLOCK", stateless = false - }, + }, + { + description = "Allow SSH Traffic to Control Plane nodes " + protocol = local.tcp_protocol, + port = -1, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + } ] - # # workers - # workers_egress = [ - # { - # description = "Allow ICMP traffic for path discovery", - # destination = local.anywhere - # destination_type = "CIDR_BLOCK", - # protocol = local.icmp_protocol, - # port = -1, - # stateless = false - # }, - # { - # description = "Allow worker nodes to communicate with OKE", - # destination = local.osn, - # destination_type = "SERVICE_CIDR_BLOCK", - # protocol = local.tcp_protocol, - # port = -1, - # stateless = false - # }, - # { - # description = "Allow worker nodes to control plane API endpoint communication", - # destination = local.cp_subnet, - # destination_type = "CIDR_BLOCK", - # protocol = local.tcp_protocol, - # port = 6443, - # stateless = false - # }, - # { - # description = "Allow worker nodes to control plane communication", - # destination = local.cp_subnet, - # destination_type = "CIDR_BLOCK", - # protocol = local.tcp_protocol, - # port = 12250, - # stateless = false - # } - # ] - - # workers_ingress = [ - # { - # description = "Allow ingress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet", - # protocol = local.all_protocols, - # port = -1, - # source = local.workers_subnet, - # source_type = "CIDR_BLOCK", - # stateless = false - # }, - # { - # description = "Allow control plane to communicate with worker nodes", - # protocol = local.tcp_protocol, - # port = 10250, - # source = local.cp_subnet, - # source_type = "CIDR_BLOCK", - # stateless = false - # }, - # { - # description = "Allow path discovery from worker nodes" - # protocol = local.icmp_protocol, - # port = -1, - # //this should be local.worker_subnet? - # source = local.anywhere, - # source_type = "CIDR_BLOCK", - # stateless = false - # } - # ] - - # int_lb_egress = [ - # { - # description = "Allow stateful egress to workers. Required for NodePorts", - # destination = local.workers_subnet, - # destination_type = "CIDR_BLOCK", - # protocol = local.tcp_protocol, - # port = "30000-32767", - # stateless = false - # }, - # { - # description = "Allow ICMP traffic for path discovery to worker nodes", - # destination = local.workers_subnet, - # destination_type = "CIDR_BLOCK", - # protocol = local.icmp_protocol, - # port = -1, - # stateless = false - # }, - # { - # description = "Allow stateful egress to workers. Required for load balancer http/tcp health checks", - # destination = local.workers_subnet, - # destination_type = "CIDR_BLOCK", - # protocol = local.tcp_protocol, - # port = local.health_check_port, - # stateless = false - # }, - # ] - - # # Combine supplied allow list and the public load balancer subnet - # internal_lb_allowed_cidrs = var.load_balancers == "both" ? concat(var.internal_lb_allowed_cidrs, tolist([local.pub_lb_subnet])) : var.internal_lb_allowed_cidrs - - # # Create a Cartesian product of allowed cidrs and ports - # internal_lb_allowed_cidrs_and_ports = setproduct(local.internal_lb_allowed_cidrs, var.internal_lb_allowed_ports) + # workers + workers_egress = [ + { + description = "Allow all egress traffic from workers", + destination = local.anywhere + destination_type = "CIDR_BLOCK", + protocol = local.all_protocols, + port = -1, + stateless = false + }, + ] - # pub_lb_egress = [ - # # { - # # description = "Allow stateful egress to internal load balancers subnet on port 80", - # # destination = local.int_lb_subnet, - # # destination_type = "CIDR_BLOCK", - # # protocol = local.tcp_protocol, - # # port = 80 - # # stateless = false - # # }, - # # { - # # description = "Allow stateful egress to internal load balancers subnet on port 443", - # # destination = local.int_lb_subnet, - # # destination_type = "CIDR_BLOCK", - # # protocol = local.tcp_protocol, - # # port = 443 - # # stateless = false - # # }, - # { - # description = "Allow stateful egress to workers. Required for NodePorts", - # destination = local.workers_subnet, - # destination_type = "CIDR_BLOCK", - # protocol = local.tcp_protocol, - # port = "30000-32767", - # stateless = false - # }, - # { - # description = "Allow ICMP traffic for path discovery to worker nodes", - # destination = local.workers_subnet, - # destination_type = "CIDR_BLOCK", - # protocol = local.icmp_protocol, - # port = -1, - # stateless = false - # }, - # ] + workers_ingress = [ + { + description = "Allow incoming traffic from service load balancers (NodePort Communication)", + protocol = local.tcp_protocol, + port = 32000 - 32767, + source = local.service-lb-int-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow incoming traffic from service load balancers (NodePort Communication)", + protocol = local.tcp_protocol, + port = 32000 - 32767, + source = local.service-lb-pub-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow control plane to worker node (Kubelet Communication)", + protocol = local.tcp_protocol, + port = 10250, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow worker to worker node (Kubelet Communication)", + protocol = local.tcp_protocol, + port = 10250, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Antrea Service communication from control plane" + protocol = local.tcp_protocol, + port = 10349, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Antrea Service communication from workers" + protocol = local.tcp_protocol, + port = 10349, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Geneve Service communication from control plane" + protocol = local.udp_protocol, + port = 6081, + source = local.cp-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Geneve Service communication from workers" + protocol = local.udp_protocol, + port = 6081, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow Path discovery" + protocol = local.icmp_protocol, + port = -1, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + }, + { + description = "Allow SSH Traffic to worker nodes " + protocol = local.tcp_protocol, + port = 22, + source = local.workers-subnet, + source_type = "CIDR_BLOCK", + stateless = false + } + ] - # public_lb_allowed_cidrs = var.public_lb_allowed_cidrs - # public_lb_allowed_cidrs_and_ports = setproduct(local.public_lb_allowed_cidrs, var.public_lb_allowed_ports) + pub_lb_egress = [ + { + description = "Allow stateful egress to workers. Required for NodePorts", + destination = local.workers-subnet, + destination_type = "CIDR_BLOCK", + protocol = local.tcp_protocol, + port = "30000-32767", + stateless = false + }, + { + description = "Allow ICMP traffic for path discovery to worker nodes", + destination = local.workers-subnet, + destination_type = "CIDR_BLOCK", + protocol = local.icmp_protocol, + port = -1, + stateless = false + }, + ] - + public_lb_allowed_cidrs = var.public_lb_allowed_cidrs + public_lb_allowed_cidrs_and_ports = setproduct(local.public_lb_allowed_cidrs, var.public_lb_allowed_ports) - } diff --git a/modules/antrea/nsgs.tf b/modules/antrea/nsgs.tf index 5b7bf86..5f82e29 100644 --- a/modules/antrea/nsgs.tf +++ b/modules/antrea/nsgs.tf @@ -1,636 +1,326 @@ # Copyright (c) 2022 Oracle Corporation and/or affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl -# # control plane nsg and rules -# resource "oci_core_network_security_group" "cp" { -# compartment_id = var.compartment_id -# display_name = var.label_prefix == "none" ? "control-plane" : "${var.label_prefix}-control-plane" -# vcn_id = var.vcn_id -# } - -# resource "oci_core_network_security_group_security_rule" "cp_egress" { -# network_security_group_id = oci_core_network_security_group.cp.id -# description = local.cp_egress[count.index].description -# destination = local.cp_egress[count.index].destination -# destination_type = local.cp_egress[count.index].destination_type -# direction = "EGRESS" -# protocol = local.cp_egress[count.index].protocol - -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.cp_egress[count.index].protocol == local.tcp_protocol && local.cp_egress[count.index].port != -1 ? [1] : [] -# content { -# destination_port_range { -# min = local.cp_egress[count.index].port -# max = local.cp_egress[count.index].port -# } -# } -# } - -# dynamic "icmp_options" { -# for_each = local.cp_egress[count.index].protocol == local.icmp_protocol ? [1] : [] -# content { -# type = 3 -# code = 4 -# } -# } - -# count = length(local.cp_egress) -# } - -# resource "oci_core_network_security_group_security_rule" "cp_egress_npn" { -# network_security_group_id = oci_core_network_security_group.cp.id -# description = "Allow Kubernetes Control plane to communicate with pods" -# destination = local.pods_subnet -# destination_type = "CIDR_BLOCK" -# direction = "EGRESS" -# protocol = local.all_protocols - -# stateless = false - -# count = var.cni_type == "npn" ? 1 :0 - -# } - -# resource "oci_core_network_security_group_security_rule" "cp_ingress" { -# network_security_group_id = oci_core_network_security_group.cp.id -# description = local.cp_ingress[count.index].description -# direction = "INGRESS" -# protocol = local.cp_ingress[count.index].protocol -# source = local.cp_ingress[count.index].source -# source_type = local.cp_ingress[count.index].source_type - -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.cp_ingress[count.index].protocol == local.tcp_protocol ? [1] : [] -# content { -# destination_port_range { -# min = local.cp_ingress[count.index].port -# max = local.cp_ingress[count.index].port -# } -# } -# } - -# dynamic "icmp_options" { -# for_each = local.cp_ingress[count.index].protocol == local.icmp_protocol ? [1] : [] -# content { -# type = 3 -# code = 4 -# } -# } - -# count = length(local.cp_ingress) - -# } - -# resource "oci_core_network_security_group_security_rule" "cp_ingress_additional_cidrs" { -# network_security_group_id = oci_core_network_security_group.cp.id -# description = "Allow additional CIDR block access to control plane. Required for kubectl/helm." -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = element(var.control_plane_allowed_cidrs, count.index) -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = 6443 -# max = 6443 -# } -# } - -# icmp_options { -# type = 3 -# code = 4 -# } - -# count = length(var.control_plane_allowed_cidrs) - -# } - -# # workers nsg and rules -# resource "oci_core_network_security_group" "workers" { -# compartment_id = var.compartment_id -# display_name = var.label_prefix == "none" ? "workers" : "${var.label_prefix}-workers" -# vcn_id = var.vcn_id -# } - -# resource "oci_core_network_security_group_security_rule" "workers_egress" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = local.workers_egress[count.index].description -# destination = local.workers_egress[count.index].destination -# destination_type = local.workers_egress[count.index].destination_type -# direction = "EGRESS" -# protocol = local.workers_egress[count.index].protocol - -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.workers_egress[count.index].protocol == local.tcp_protocol && local.workers_egress[count.index].port != -1 ? [1] : [] -# content { -# destination_port_range { -# min = local.workers_egress[count.index].port -# max = local.workers_egress[count.index].port -# } -# } -# } - -# dynamic "icmp_options" { -# for_each = local.workers_egress[count.index].protocol == local.icmp_protocol ? [1] : [] -# content { -# type = 3 -# code = 4 -# } -# } - -# count = length(local.workers_egress) -# } - -# resource "oci_core_network_security_group_security_rule" "workers_egress_flannel" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow egress for all traffic to allow pods to communicate between each other on different worker nodes on the worker subnet" -# destination = local.workers_subnet -# destination_type = "CIDR_BLOCK" -# direction = "EGRESS" -# protocol = local.all_protocols - -# stateless = false - -# count = var.cni_type == "flannel" ? 1: 0 -# } - -# resource "oci_core_network_security_group_security_rule" "workers_egress_npn" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow worker nodes access to pods" -# destination = local.pods_subnet -# destination_type = "CIDR_BLOCK" -# direction = "EGRESS" -# protocol = local.all_protocols - -# stateless = false - -# count = var.cni_type == "npn" ? 1: 0 -# } - -# # add this rule separately so it can be controlled independently -# resource "oci_core_network_security_group_security_rule" "workers_egress_internet" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow worker nodes access to Internet. Required for getting container images or using external services" -# destination = local.anywhere -# destination_type = "CIDR_BLOCK" -# direction = "EGRESS" -# protocol = local.tcp_protocol - -# stateless = false - -# count = var.allow_worker_internet_access == true ? 1 : 0 - -# } - -# resource "oci_core_network_security_group_security_rule" "workers_ingress" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = local.workers_ingress[count.index].description -# direction = "INGRESS" -# protocol = local.workers_ingress[count.index].protocol -# source = local.workers_ingress[count.index].source -# source_type = local.workers_ingress[count.index].source_type - -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.workers_ingress[count.index].protocol == local.tcp_protocol && local.workers_ingress[count.index].port != -1 ? [1] : [] -# content { -# destination_port_range { -# min = local.workers_ingress[count.index].port -# max = local.workers_ingress[count.index].port -# } -# } -# } - -# dynamic "icmp_options" { -# for_each = local.workers_ingress[count.index].protocol == local.icmp_protocol ? [1] : [] -# content { -# type = 3 -# code = 4 -# } -# } - -# count = length(local.workers_ingress) - -# } - -# # add the next 4 rules separately so it can be controlled independently based on which lbs are created -# resource "oci_core_network_security_group_security_rule" "workers_ingress_from_int_lb" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow internal load balancers traffic to workers" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = local.int_lb_subnet -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = local.node_port_min -# max = local.node_port_max -# } -# } - -# count = var.load_balancers == "internal" || var.load_balancers == "both" ? 1 : 0 - -# } - -# resource "oci_core_network_security_group_security_rule" "workers_healthcheck_ingress_from_int_lb" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow internal load balancers health check to workers" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = local.int_lb_subnet -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = local.health_check_port -# max = local.health_check_port -# } -# } - -# count = var.load_balancers == "internal" || var.load_balancers == "both" ? 1 : 0 - -# } - -# resource "oci_core_network_security_group_security_rule" "workers_ingress_from_pub_lb" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow public load balancers traffic to workers" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = local.pub_lb_subnet -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = local.node_port_min -# max = local.node_port_max -# } -# } - -# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 - -# } - -# resource "oci_core_network_security_group_security_rule" "workers_healthcheck_ingress_from_pub_lb" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow public load balancers health check to workers" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = local.pub_lb_subnet -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = local.health_check_port -# max = local.health_check_port -# } -# } - -# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 - -# } - -# resource "oci_core_network_security_group_security_rule" "workers_ssh_ingress_from_bastion" { -# network_security_group_id = oci_core_network_security_group.workers.id -# description = "Allow ssh access to workers via Bastion host" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = local.bastion_subnet -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = local.ssh_port -# max = local.ssh_port -# } -# } - -# count = var.allow_worker_ssh_access == true ? 1 : 0 - -# } - -# # pod nsg and rules -# resource "oci_core_network_security_group" "pods" { -# compartment_id = var.compartment_id -# display_name = var.label_prefix == "none" ? "pods" : "${var.label_prefix}-pods" -# vcn_id = var.vcn_id -# } - -# resource "oci_core_network_security_group_security_rule" "pods_egress" { -# network_security_group_id = oci_core_network_security_group.pods.id -# description = local.pods_egress[count.index].description -# destination = local.pods_egress[count.index].destination -# destination_type = local.pods_egress[count.index].destination_type -# direction = "EGRESS" -# protocol = local.pods_egress[count.index].protocol - -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.pods_egress[count.index].protocol == local.tcp_protocol && local.pods_egress[count.index].port != -1 ? [1] : [] -# content { -# destination_port_range { -# min = local.pods_egress[count.index].port -# max = local.pods_egress[count.index].port -# } -# } -# } - -# dynamic "icmp_options" { -# for_each = local.pods_egress[count.index].protocol == local.icmp_protocol ? [1] : [] -# content { -# type = 3 -# code = 4 -# } -# } - -# count = var.cni_type =="npn" ? length(local.pods_egress) : 0 -# } - -# # add this rule separately so it can be controlled independently -# resource "oci_core_network_security_group_security_rule" "pods_egress_internet" { -# network_security_group_id = oci_core_network_security_group.pods.id -# description = "Allow pods access to Internet" -# destination = local.anywhere -# destination_type = "CIDR_BLOCK" -# direction = "EGRESS" -# protocol = local.tcp_protocol - -# stateless = false -# count = (var.cni_type =="npn" && var.allow_pod_internet_access == true) ? 1 : 0 - -# } - -# # internal lb nsg and rules -# resource "oci_core_network_security_group" "int_lb" { -# compartment_id = var.compartment_id -# display_name = var.label_prefix == "none" ? "int-lb" : "${var.label_prefix}-int-lb" -# vcn_id = var.vcn_id - -# count = var.load_balancers == "internal" || var.load_balancers == "both" ? 1 : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "int_lb_egress" { -# network_security_group_id = oci_core_network_security_group.int_lb[0].id -# description = local.int_lb_egress[count.index].description -# destination = local.int_lb_egress[count.index].destination -# destination_type = local.int_lb_egress[count.index].destination_type -# direction = "EGRESS" -# protocol = local.int_lb_egress[count.index].protocol - -# stateless = false -# # TODO: condition for end-to-end SSL/SSL termination -# dynamic "tcp_options" { -# for_each = local.int_lb_egress[count.index].protocol == local.tcp_protocol && local.int_lb_egress[count.index].port != -1 ? [1] : [] -# content { -# destination_port_range { -# min = length(regexall("-", local.int_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.int_lb_egress[count.index].port), 0)) : local.int_lb_egress[count.index].port -# max = length(regexall("-", local.int_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.int_lb_egress[count.index].port), 1)) : local.int_lb_egress[count.index].port -# } -# } -# } - -# dynamic "icmp_options" { -# for_each = local.int_lb_egress[count.index].protocol == local.icmp_protocol ? [1] : [] -# content { -# type = 3 -# code = 4 -# } -# } - -# count = var.load_balancers == "internal" || var.load_balancers == "both" ? length(local.int_lb_egress) : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "int_lb_ingress" { -# network_security_group_id = oci_core_network_security_group.int_lb[0].id -# description = "Allow stateful ingress from ${element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 0)} on port ${element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1)}" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 0) -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = length(regexall("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1)), 0) : element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1) -# max = length(regexall("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1)), 1) : element(element(local.internal_lb_allowed_cidrs_and_ports, count.index), 1) -# } -# } - -# count = var.load_balancers == "internal" || var.load_balancers == "both" ? length(local.internal_lb_allowed_cidrs_and_ports) : 0 -# } - -# # public lb nsg and rules -# resource "oci_core_network_security_group" "pub_lb" { -# compartment_id = var.compartment_id -# display_name = var.label_prefix == "none" ? "pub-lb" : "${var.label_prefix}-pub-lb" -# vcn_id = var.vcn_id - -# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "pub_lb_egress" { -# network_security_group_id = oci_core_network_security_group.pub_lb[0].id -# description = local.pub_lb_egress[count.index].description -# destination = local.pub_lb_egress[count.index].destination -# destination_type = local.pub_lb_egress[count.index].destination_type -# direction = "EGRESS" -# protocol = local.pub_lb_egress[count.index].protocol - -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.pub_lb_egress[count.index].protocol == local.tcp_protocol && local.pub_lb_egress[count.index].port != -1 ? [1] : [] -# content { -# destination_port_range { -# min = length(regexall("-", local.pub_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.pub_lb_egress[count.index].port), 0)) : local.pub_lb_egress[count.index].port -# max = length(regexall("-", local.pub_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.pub_lb_egress[count.index].port), 1)) : local.pub_lb_egress[count.index].port -# } -# } -# } - -# dynamic "icmp_options" { -# for_each = local.pub_lb_egress[count.index].protocol == local.icmp_protocol ? [1] : [] -# content { -# type = 3 -# code = 4 -# } -# } - -# count = var.load_balancers == "public" || var.load_balancers == "both" ? length(local.pub_lb_egress) : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "pub_lb_egress_health_check_to_workers" { -# network_security_group_id = oci_core_network_security_group.pub_lb[0].id -# description = "Allow public load balancer health checks to workers" -# destination = local.workers_subnet -# destination_type = "CIDR_BLOCK" -# direction = "EGRESS" -# protocol = local.tcp_protocol - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = local.health_check_port -# max = local.health_check_port -# } -# } - -# count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "pub_lb_egress_health_check_to_int_lb" { -# network_security_group_id = oci_core_network_security_group.pub_lb[0].id -# description = "Allow public load balancer health checks to internal load balancers" -# destination = local.int_lb_subnet -# destination_type = "CIDR_BLOCK" -# direction = "EGRESS" -# protocol = local.tcp_protocol - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = length(regexall("-", element(var.internal_lb_allowed_ports, count.index))) > 0 ? tonumber(element(split("-", element(var.internal_lb_allowed_ports, count.index)), 0)) : element(var.internal_lb_allowed_ports, count.index) -# max = length(regexall("-", element(var.internal_lb_allowed_ports, count.index))) > 0 ? tonumber(element(split("-", element(var.internal_lb_allowed_ports, count.index)), 1)) : element(var.internal_lb_allowed_ports, count.index) -# } -# } - -# count = var.load_balancers == "both" ? length(var.internal_lb_allowed_ports) : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "pub_lb_ingress" { -# network_security_group_id = oci_core_network_security_group.pub_lb[0].id -# description = "Allow stateful ingress from ${element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 0)} on port ${element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)}" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 0) -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = length(regexall("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)), 0) : element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1) -# max = length(regexall("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)), 1) : element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1) -# } -# } - -# count = var.load_balancers == "public" || var.load_balancers == "both" ? length(local.public_lb_allowed_cidrs_and_ports) : 0 -# } - -# # waf lb nsg and rules -# resource "oci_core_network_security_group" "waf" { -# compartment_id = var.compartment_id -# display_name = var.label_prefix == "none" ? "waf" : "${var.label_prefix}-waf" -# vcn_id = var.vcn_id - -# count = var.enable_waf == true ? 1 : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "waf_ingress" { -# for_each = var.enable_waf == true ? toset(local.waf_cidr_list) : toset([]) -# network_security_group_id = oci_core_network_security_group.waf[0].id -# description = "Allow stateful ingress from WAF" -# direction = "INGRESS" -# protocol = local.tcp_protocol -# source = each.key -# source_type = "CIDR_BLOCK" - -# stateless = false - -# tcp_options { -# destination_port_range { -# min = 443 -# max = 443 -# } -# } - -# } - -# ## fss : instance network security group rules - -# resource "oci_core_network_security_group_security_rule" "fss_inst_ingress" { -# network_security_group_id = oci_core_network_security_group.workers.id -# direction = "INGRESS" -# protocol = local.fss_inst_ingress[count.index].protocol -# source = local.fss_inst_ingress[count.index].source -# source_type = local.fss_inst_ingress[count.index].source_type -# description = local.fss_inst_ingress[count.index].description -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.fss_inst_ingress[count.index].protocol == local.tcp_protocol ? [1] : [] -# content { -# source_port_range { -# min = local.fss_inst_ingress[count.index].port -# max = local.fss_inst_ingress[count.index].port -# } -# } -# } - -# dynamic "udp_options" { -# for_each = local.fss_inst_ingress[count.index].protocol == local.udp_protocol ? [1] : [] -# content { -# source_port_range { -# min = local.fss_inst_ingress[count.index].port -# max = local.fss_inst_ingress[count.index].port -# } -# } -# } - -# count = var.create_fss ? length(local.fss_inst_ingress) : 0 -# } - -# resource "oci_core_network_security_group_security_rule" "fss_inst_egress" { -# network_security_group_id = oci_core_network_security_group.workers.id -# direction = "EGRESS" -# protocol = local.fss_inst_egress[count.index].protocol -# destination = local.fss_inst_egress[count.index].destination -# destination_type = local.fss_inst_egress[count.index].destination_type -# description = local.fss_inst_egress[count.index].description -# stateless = false - -# dynamic "tcp_options" { -# for_each = local.fss_inst_egress[count.index].protocol == local.tcp_protocol ? [1] : [] -# content { -# destination_port_range { -# min = local.fss_inst_egress[count.index].port -# max = local.fss_inst_egress[count.index].port -# } -# } -# } - -# dynamic "udp_options" { -# for_each = local.fss_inst_egress[count.index].protocol == local.udp_protocol ? [1] : [] -# content { -# destination_port_range { -# min = local.fss_inst_egress[count.index].port -# max = local.fss_inst_egress[count.index].port -# } -# } -# } - -# count = var.create_fss ? length(local.fss_inst_egress) : 0 -# } +# control plane nsg and rules +resource "oci_core_network_security_group" "cp" { + compartment_id = var.compartment_id + display_name = var.label_prefix == "none" ? "control-plane" : "${var.label_prefix}-control-plane" + vcn_id = var.vcn_id +} + +resource "oci_core_network_security_group_security_rule" "cp_egress" { + network_security_group_id = oci_core_network_security_group.cp.id + description = local.cp_egress[count.index].description + destination = local.cp_egress[count.index].destination + destination_type = local.cp_egress[count.index].destination_type + direction = "EGRESS" + protocol = local.cp_egress[count.index].protocol + + stateless = false + + dynamic "tcp_options" { + for_each = local.cp_egress[count.index].protocol == local.tcp_protocol && local.cp_egress[count.index].port != -1 ? [1] : [] + content { + destination_port_range { + min = local.cp_egress[count.index].port + max = local.cp_egress[count.index].port + } + } + } + + dynamic "icmp_options" { + for_each = local.cp_egress[count.index].protocol == local.icmp_protocol ? [1] : [] + content { + type = 3 + code = 4 + } + } + + count = length(local.cp_egress) +} + +resource "oci_core_network_security_group_security_rule" "cp_ingress" { + network_security_group_id = oci_core_network_security_group.cp.id + description = local.cp_ingress[count.index].description + direction = "INGRESS" + protocol = local.cp_ingress[count.index].protocol + source = local.cp_ingress[count.index].source + source_type = local.cp_ingress[count.index].source_type + + stateless = false + + dynamic "tcp_options" { + for_each = local.cp_ingress[count.index].protocol == local.tcp_protocol ? [1] : [] + content { + destination_port_range { + min = local.cp_ingress[count.index].port + max = local.cp_ingress[count.index].port + } + } + } + + dynamic "icmp_options" { + for_each = local.cp_ingress[count.index].protocol == local.icmp_protocol ? [1] : [] + content { + type = 3 + code = 4 + } + } + + count = length(local.cp_ingress) + +} + +resource "oci_core_network_security_group_security_rule" "cp_ingress_additional_cidrs" { + network_security_group_id = oci_core_network_security_group.cp.id + description = "Allow additional CIDR block access to control plane. Required for kubectl/helm." + direction = "INGRESS" + protocol = local.tcp_protocol + source = element(var.control_plane_allowed_cidrs, count.index) + source_type = "CIDR_BLOCK" + + stateless = false + + tcp_options { + destination_port_range { + min = 6443 + max = 6443 + } + } + + icmp_options { + type = 3 + code = 4 + } + + count = length(var.control_plane_allowed_cidrs) + +} + +# workers nsg and rules +resource "oci_core_network_security_group" "workers" { + compartment_id = var.compartment_id + display_name = var.label_prefix == "none" ? "workers" : "${var.label_prefix}-workers" + vcn_id = var.vcn_id +} + +resource "oci_core_network_security_group_security_rule" "workers_egress" { + network_security_group_id = oci_core_network_security_group.workers.id + description = local.workers_egress[count.index].description + destination = local.workers_egress[count.index].destination + destination_type = local.workers_egress[count.index].destination_type + direction = "EGRESS" + protocol = local.workers_egress[count.index].protocol + + stateless = false + + dynamic "tcp_options" { + for_each = local.workers_egress[count.index].protocol == local.tcp_protocol && local.workers_egress[count.index].port != -1 ? [1] : [] + content { + destination_port_range { + min = local.workers_egress[count.index].port + max = local.workers_egress[count.index].port + } + } + } + + dynamic "icmp_options" { + for_each = local.workers_egress[count.index].protocol == local.icmp_protocol ? [1] : [] + content { + type = 3 + code = 4 + } + } + + count = length(local.workers_egress) +} + +# add this rule separately so it can be controlled independently +resource "oci_core_network_security_group_security_rule" "workers_egress_internet" { + network_security_group_id = oci_core_network_security_group.workers.id + description = "Allow worker nodes access to Internet. Required for getting container images or using external services" + destination = local.anywhere + destination_type = "CIDR_BLOCK" + direction = "EGRESS" + protocol = local.tcp_protocol + + stateless = false + + count = var.allow_worker_internet_access == true ? 1 : 0 + +} + +resource "oci_core_network_security_group_security_rule" "workers_ingress" { + network_security_group_id = oci_core_network_security_group.workers.id + description = local.workers_ingress[count.index].description + direction = "INGRESS" + protocol = local.workers_ingress[count.index].protocol + source = local.workers_ingress[count.index].source + source_type = local.workers_ingress[count.index].source_type + + stateless = false + + dynamic "tcp_options" { + for_each = local.workers_ingress[count.index].protocol == local.tcp_protocol && local.workers_ingress[count.index].port != -1 ? [1] : [] + content { + destination_port_range { + min = local.workers_ingress[count.index].port + max = local.workers_ingress[count.index].port + } + } + } + + dynamic "icmp_options" { + for_each = local.workers_ingress[count.index].protocol == local.icmp_protocol ? [1] : [] + content { + type = 3 + code = 4 + } + } + + count = length(local.workers_ingress) + +} + +resource "oci_core_network_security_group_security_rule" "workers_ingress_from_pub_lb" { + network_security_group_id = oci_core_network_security_group.workers.id + description = "Allow public load balancers traffic to workers" + direction = "INGRESS" + protocol = local.tcp_protocol + source = local.service-lb-pub-subnet + source_type = "CIDR_BLOCK" + + stateless = false + + tcp_options { + destination_port_range { + min = local.node_port_min + max = local.node_port_max + } + } + + count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 + +} + +resource "oci_core_network_security_group_security_rule" "workers_healthcheck_ingress_from_pub_lb" { + network_security_group_id = oci_core_network_security_group.workers.id + description = "Allow public load balancers health check to workers" + direction = "INGRESS" + protocol = local.tcp_protocol + source = local.service-lb-pub-subnet + source_type = "CIDR_BLOCK" + + stateless = false + + tcp_options { + destination_port_range { + min = local.health_check_port + max = local.health_check_port + } + } + + count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 + +} + +resource "oci_core_network_security_group_security_rule" "workers_ssh_ingress_from_bastion" { + network_security_group_id = oci_core_network_security_group.workers.id + description = "Allow ssh access to workers via Bastion host" + direction = "INGRESS" + protocol = local.tcp_protocol + source = local.bastion-subnet + source_type = "CIDR_BLOCK" + + stateless = false + + tcp_options { + destination_port_range { + min = local.ssh_port + max = local.ssh_port + } + } + + count = var.allow_worker_ssh_access == true ? 1 : 0 + +} + +# public lb nsg and rules +resource "oci_core_network_security_group" "pub_lb" { + compartment_id = var.compartment_id + display_name = var.label_prefix == "none" ? "pub-lb" : "${var.label_prefix}-pub-lb" + vcn_id = var.vcn_id + + count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 +} + +resource "oci_core_network_security_group_security_rule" "pub_lb_egress" { + network_security_group_id = oci_core_network_security_group.pub_lb[0].id + description = local.pub_lb_egress[count.index].description + destination = local.pub_lb_egress[count.index].destination + destination_type = local.pub_lb_egress[count.index].destination_type + direction = "EGRESS" + protocol = local.pub_lb_egress[count.index].protocol + + stateless = false + + dynamic "tcp_options" { + for_each = local.pub_lb_egress[count.index].protocol == local.tcp_protocol && local.pub_lb_egress[count.index].port != -1 ? [1] : [] + content { + destination_port_range { + min = length(regexall("-", local.pub_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.pub_lb_egress[count.index].port), 0)) : local.pub_lb_egress[count.index].port + max = length(regexall("-", local.pub_lb_egress[count.index].port)) > 0 ? tonumber(element(split("-", local.pub_lb_egress[count.index].port), 1)) : local.pub_lb_egress[count.index].port + } + } + } + + dynamic "icmp_options" { + for_each = local.pub_lb_egress[count.index].protocol == local.icmp_protocol ? [1] : [] + content { + type = 3 + code = 4 + } + } + + count = var.load_balancers == "public" || var.load_balancers == "both" ? length(local.pub_lb_egress) : 0 +} + +resource "oci_core_network_security_group_security_rule" "pub_lb_egress_health_check_to_workers" { + network_security_group_id = oci_core_network_security_group.pub_lb[0].id + description = "Allow public load balancer health checks to workers" + destination = local.workers-subnet + destination_type = "CIDR_BLOCK" + direction = "EGRESS" + protocol = local.tcp_protocol + + stateless = false + + tcp_options { + destination_port_range { + min = local.health_check_port + max = local.health_check_port + } + } + + count = var.load_balancers == "public" || var.load_balancers == "both" ? 1 : 0 +} + +resource "oci_core_network_security_group_security_rule" "pub_lb_ingress" { + network_security_group_id = oci_core_network_security_group.pub_lb[0].id + description = "Allow stateful ingress from ${element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 0)} on port ${element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)}" + direction = "INGRESS" + protocol = local.tcp_protocol + source = element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 0) + source_type = "CIDR_BLOCK" + + stateless = false + + tcp_options { + destination_port_range { + min = length(regexall("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)), 0) : element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1) + max = length(regexall("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1))) > 0 ? element(split("-", element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1)), 1) : element(element(local.public_lb_allowed_cidrs_and_ports, count.index), 1) + } + } + + count = var.load_balancers == "public" || var.load_balancers == "both" ? length(local.public_lb_allowed_cidrs_and_ports) : 0 +} diff --git a/modules/antrea/variables.tf b/modules/antrea/variables.tf index 7c997b9..683d036 100644 --- a/modules/antrea/variables.tf +++ b/modules/antrea/variables.tf @@ -32,27 +32,22 @@ variable "allow_worker_internet_access" { type = bool } -variable "worker_type" {} - -# # load balancers -# variable "load_balancers" { -# type = string -# } +variable "allow_worker_ssh_access" { + type = bool +} -# # internal load balancers -# variable "internal_lb_allowed_cidrs" { -# type = list(any) -# } +variable "worker_type" {} -# variable "internal_lb_allowed_ports" { -# type = list(any) -# } +# load balancers +variable "load_balancers" { + type = string +} -# # public load balancers -# variable "public_lb_allowed_cidrs" { -# type = list(any) -# } +# public load balancers +variable "public_lb_allowed_cidrs" { + type = list(any) +} -# variable "public_lb_allowed_ports" { -# type = list(any) -# } \ No newline at end of file +variable "public_lb_allowed_ports" { + type = list(any) +} diff --git a/variables.tf b/variables.tf index 3929f5d..f8008bb 100644 --- a/variables.tf +++ b/variables.tf @@ -239,6 +239,28 @@ variable "load_balancers" { } } +variable "public_lb_allowed_cidrs" { + default = ["0.0.0.0/0"] + description = "The list of CIDR blocks from which the public load balancer can be accessed." + type = list(string) + + validation { + condition = length(var.public_lb_allowed_cidrs) > 0 + error_message = "At least 1 CIDR block is required." + } +} + +variable "public_lb_allowed_ports" { + default = [443] + description = "List of allowed ports for public load balancers." + type = list(any) + + validation { + condition = length(var.public_lb_allowed_ports) > 0 + error_message = "At least 1 port is required." + } +} + # workers variable "worker_type" { default = "private" @@ -250,6 +272,12 @@ variable "worker_type" { } } +variable "allow_worker_ssh_access" { + default = false + description = "Whether to allow ssh access to worker nodes." + type = bool +} + # tagging variable "freeform_tags" { default = {