diff --git a/app-dev/devops-and-containers/oke/README.md b/app-dev/devops-and-containers/oke/README.md index 9e25dd69e..0f4db3b71 100644 --- a/app-dev/devops-and-containers/oke/README.md +++ b/app-dev/devops-and-containers/oke/README.md @@ -54,9 +54,11 @@ Reviewed: 20.12.2023 - [OKE policies](./oke-policies/policies.md) # Reusable Assets Overview - + +- [OKE Resource Manager QuickStart](https://github.com/oracle-devrel/technology-engineering/tree/main/app-dev/oke/oke-rm) +- [OKE GitOps Solution](https://github.com/oracle-devrel/technology-engineering/tree/main/app-dev/oke/oke-gitops) +- [OKE Node Packer Solution](https://github.com/oracle-devrel/technology-engineering/tree/main/app-dev/oke/oke-node-packer) - [Cluster Api OCI](https://github.com/oracle-devrel/technology-engineering/tree/main/app-dev/oke/capoci) -- [Cloud Native QuickStart](https://github.com/alcampag/oci-cn-quickstart) # License diff --git a/app-dev/devops-and-containers/oke/oke-rm/README.md b/app-dev/devops-and-containers/oke/oke-rm/README.md index 16bc38951..ddcf149c5 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/README.md +++ b/app-dev/devops-and-containers/oke/oke-rm/README.md @@ -17,13 +17,13 @@ This stack is used to create the initial network infrastructure for OKE. When co * You can apply this stack even on an existing VCN, so that only the NSGs for OKE will be created * The default CNI is the VCN Native CNI, and it is the recommended one -[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.1.9/infra.zip) +[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.2.0/infra.zip) ## Step 2: Create the OKE control plane This stack is used to create the OKE control plane ONLY. -[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.1.9/oke.zip) +[![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-devrel/technology-engineering/releases/download/oke-rm-1.2.0/oke.zip) Also note that if the network infrastructure is located in a different compartment than the OKE cluster AND you are planning to use the OCI_VCN_NATIVE CNI, you must add these policies: diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/infra.zip b/app-dev/devops-and-containers/oke/oke-rm/infra/infra.zip index b6e094ee6..e7508684a 100644 Binary files a/app-dev/devops-and-containers/oke/oke-rm/infra/infra.zip and b/app-dev/devops-and-containers/oke/oke-rm/infra/infra.zip differ diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/local.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/local.tf index 5a31b1a60..9869af864 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/local.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/local.tf @@ -1,25 +1,25 @@ locals { # VCN_NATIVE_CNI internally it is mapped as npn - cni = var.cni_type == "vcn_native" ? "npn" : var.cni_type + cni = var.cni_type == "vcn_native" ? "npn" : var.cni_type vcn_cidr_blocks = [var.vcn_cidr_block] subnets = { cidr = { - pod = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 1, 0) : null # e.g., "10.1.0.0/17" - worker = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 3, 4) : null # e.g., "10.1.128.0/19" - lb_external = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 8, 160) : null # e.g., "10.1.160.0/24" - lb_internal = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 8, 161) : null # e.g., "10.1.161.0/24" - fss = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 8, 162) : null # e.g., "10.1.162.0/24" - bastion = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 13, 5216) : null # e.g., "10.1.163.0/29" - cp = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 13, 5217) : null # e.g., "10.1.163.8/29" + pod = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 1, 0) : null # e.g., "10.1.0.0/17" + worker = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 3, 4) : null # e.g., "10.1.128.0/19" + lb_external = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 8, 160) : null # e.g., "10.1.160.0/24" + lb_internal = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 8, 161) : null # e.g., "10.1.161.0/24" + fss = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 8, 162) : null # e.g., "10.1.162.0/24" + bastion = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 13, 5216) : null # e.g., "10.1.163.0/29" + cp = var.create_vcn ? cidrsubnet(var.vcn_cidr_block, 13, 5217) : null # e.g., "10.1.163.8/29" } dns = { - pod = "pod" - worker = "worker" + pod = "pod" + worker = "worker" lb_external = "lbext" lb_internal = "lbint" - fss = "fss" - bastion = "bastion" - cp = "cp" + fss = "fss" + bastion = "bastion" + cp = "cp" } } } diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/main.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/main.tf index 7e9fcaed3..809238d15 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/main.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/main.tf @@ -1,64 +1,64 @@ module "network" { - source = "./modules/network" + source = "./modules/network" network_compartment_id = var.network_compartment_id - region = var.region - cni_type = local.cni + region = var.region + cni_type = local.cni # VCN - create_vcn = var.create_vcn - vcn_id = var.vcn_id - vcn_name = var.vcn_name + create_vcn = var.create_vcn + vcn_id = var.vcn_id + vcn_name = var.vcn_name vcn_cidr_blocks = local.vcn_cidr_blocks - vcn_dns_label = var.vcn_dns_label + vcn_dns_label = var.vcn_dns_label # CP SUBNET - create_cp_subnet = var.create_cp_subnet - cp_subnet_cidr = local.subnets.cidr.cp - cp_subnet_dns_label = local.subnets.dns.cp - cp_subnet_name = var.cp_subnet_name - cp_subnet_private = var.cp_subnet_private + create_cp_subnet = var.create_cp_subnet + cp_subnet_cidr = local.subnets.cidr.cp + cp_subnet_dns_label = local.subnets.dns.cp + cp_subnet_name = var.cp_subnet_name + cp_subnet_private = var.cp_subnet_private cp_allowed_source_cidr = var.cp_allowed_source_cidr # LB SUBNETS - create_external_lb_subnet = var.create_external_lb_subnet - external_lb_cidr = local.subnets.cidr.lb_external + create_external_lb_subnet = var.create_external_lb_subnet + external_lb_cidr = local.subnets.cidr.lb_external external_lb_subnet_dns_label = local.subnets.dns.lb_external - external_lb_subnet_name = var.external_lb_subnet_name - create_internal_lb_subnet = var.create_internal_lb_subnet - internal_lb_cidr = local.subnets.cidr.lb_internal + external_lb_subnet_name = var.external_lb_subnet_name + create_internal_lb_subnet = var.create_internal_lb_subnet + internal_lb_cidr = local.subnets.cidr.lb_internal internal_lb_subnet_dns_label = local.subnets.dns.lb_internal - internal_lb_subnet_name = var.internal_lb_subnet_name + internal_lb_subnet_name = var.internal_lb_subnet_name # WORKER SUBNET - create_worker_subnet = var.create_worker_subnet - worker_subnet_cidr = local.subnets.cidr.worker + create_worker_subnet = var.create_worker_subnet + worker_subnet_cidr = local.subnets.cidr.worker worker_subnet_dns_label = local.subnets.dns.worker - worker_subnet_name = var.worker_subnet_name + worker_subnet_name = var.worker_subnet_name # POD SUBNET - create_pod_subnet = var.create_pod_subnet - pod_subnet_cidr = local.subnets.cidr.pod + create_pod_subnet = var.create_pod_subnet + pod_subnet_cidr = local.subnets.cidr.pod pod_subnet_dns_label = local.subnets.dns.pod - pod_subnet_name = var.pod_subnet_name + pod_subnet_name = var.pod_subnet_name # BASTION SUBNET - create_bastion_subnet = var.create_bastion_subnet - bastion_subnet_cidr = local.subnets.cidr.bastion + create_bastion_subnet = var.create_bastion_subnet + bastion_subnet_cidr = local.subnets.cidr.bastion bastion_subnet_dns_label = local.subnets.dns.bastion - bastion_subnet_name = var.bastion_subnet_name - bastion_subnet_private = var.bastion_subnet_private + bastion_subnet_name = var.bastion_subnet_name + bastion_subnet_private = var.bastion_subnet_private # FSS SUBNET - create_fss = var.create_fss - fss_subnet_cidr = local.subnets.cidr.fss + create_fss = var.create_fss + fss_subnet_cidr = local.subnets.cidr.fss fss_subnet_dns_label = local.subnets.dns.fss - fss_subnet_name = var.fss_subnet_name + fss_subnet_name = var.fss_subnet_name # GATEWAYS - create_gateways = var.create_gateways + create_gateways = var.create_gateways create_internet_gateway = var.create_internet_gateway # CONTROL PLANE EXTERNAL CONNECTION - cp_external_nat = var.cp_external_nat + cp_external_nat = var.cp_external_nat allow_external_cp_traffic = var.allow_external_cp_traffic - cp_egress_cidr = var.cp_egress_cidr + cp_egress_cidr = var.cp_egress_cidr # DRG - enable_drg = var.enable_drg - create_drg = var.create_drg - drg_id = var.drg_id - drg_name = var.drg_name + enable_drg = var.enable_drg + create_drg = var.create_drg + drg_id = var.drg_id + drg_name = var.drg_name create_drg_attachment = var.create_drg_attachment - peer_vcns = var.peer_vcns + peer_vcns = var.peer_vcns } \ No newline at end of file diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/drg.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/drg.tf index f991d33d4..bf0126193 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/drg.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/drg.tf @@ -1,16 +1,16 @@ resource "oci_core_drg" "vcn_drg" { compartment_id = var.network_compartment_id - display_name = var.drg_name + display_name = var.drg_name count = local.create_drg ? 1 : 0 } resource "oci_core_drg_attachment" "oke_drg_attachment" { - drg_id = local.drg_id + drg_id = local.drg_id display_name = var.vcn_name network_details { - id = local.vcn_id + id = local.vcn_id type = "VCN" } diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/gateways.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/gateways.tf index a5b4f49ca..75921f7e3 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/gateways.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/gateways.tf @@ -1,7 +1,7 @@ resource "oci_core_service_gateway" "service_gateway" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = "SG" + display_name = "SG" services { service_id = lookup(data.oci_core_services.all_oci_services.services[0], "id") } @@ -11,13 +11,13 @@ resource "oci_core_service_gateway" "service_gateway" { resource "oci_core_nat_gateway" "nat_gateway" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = "NAT" - count = local.create_gateways ? 1 : 0 + display_name = "NAT" + count = local.create_gateways ? 1 : 0 } resource "oci_core_internet_gateway" "internet_gateway" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = "IG" - count = local.create_internet_gateway ? 1 : 0 + display_name = "IG" + count = local.create_internet_gateway ? 1 : 0 } \ No newline at end of file diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/local.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/local.tf index a7c241540..428ba5d27 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/local.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/local.tf @@ -1,29 +1,29 @@ locals { - is_npn = var.cni_type == "npn" - create_pod_subnet = var.create_pod_subnet && local.is_npn && var.create_vcn - create_cp_subnet = var.create_cp_subnet && var.create_vcn - create_bastion_subnet = var.create_bastion_subnet && var.create_vcn - create_fss_subnet = var.create_fss && var.create_vcn - create_worker_subnet = var.create_worker_subnet && var.create_vcn - create_external_lb_subnet = var.create_external_lb_subnet && var.create_vcn - create_internal_lb_subnet = var.create_internal_lb_subnet && var.create_vcn - all_subnet_private = (var.cp_subnet_private || ! local.create_cp_subnet) && (! local.create_external_lb_subnet) && (var.bastion_subnet_private || ! var.create_bastion_subnet) - vcn_id = var.create_vcn ? oci_core_vcn.spoke_vcn.0.id : var.vcn_id - service_gateway_id = var.create_gateways ? oci_core_service_gateway.service_gateway.0.id : null - nat_gateway_id = var.create_gateways ? oci_core_nat_gateway.nat_gateway.0.id : null - cp_nat_mode = local.create_cp_subnet && var.cp_subnet_private && var.cp_external_nat - create_cp_external_traffic_rule = var.allow_external_cp_traffic && (! var.create_cp_subnet || (! var.cp_subnet_private || var.cp_external_nat)) + is_npn = var.cni_type == "npn" + create_pod_subnet = var.create_pod_subnet && local.is_npn && var.create_vcn + create_cp_subnet = var.create_cp_subnet && var.create_vcn + create_bastion_subnet = var.create_bastion_subnet && var.create_vcn + create_fss_subnet = var.create_fss && var.create_vcn + create_worker_subnet = var.create_worker_subnet && var.create_vcn + create_external_lb_subnet = var.create_external_lb_subnet && var.create_vcn + create_internal_lb_subnet = var.create_internal_lb_subnet && var.create_vcn + all_subnet_private = (var.cp_subnet_private || !local.create_cp_subnet) && (!local.create_external_lb_subnet) && (var.bastion_subnet_private || !var.create_bastion_subnet) + vcn_id = var.create_vcn ? oci_core_vcn.spoke_vcn.0.id : var.vcn_id + service_gateway_id = var.create_gateways ? oci_core_service_gateway.service_gateway.0.id : null + nat_gateway_id = var.create_gateways ? oci_core_nat_gateway.nat_gateway.0.id : null + cp_nat_mode = local.create_cp_subnet && var.cp_subnet_private && var.cp_external_nat + create_cp_external_traffic_rule = var.allow_external_cp_traffic && (!var.create_cp_subnet || (!var.cp_subnet_private || var.cp_external_nat)) - create_gateways = (var.create_gateways && ! var.create_vcn) || var.create_vcn - create_internet_gateway = (local.create_gateways && ! var.create_vcn && var.create_internet_gateway) || (var.create_vcn && ! local.all_subnet_private) + create_gateways = (var.create_gateways && !var.create_vcn) || var.create_vcn + create_internet_gateway = (local.create_gateways && !var.create_vcn && var.create_internet_gateway) || (var.create_vcn && !local.all_subnet_private) - create_drg = var.enable_drg && var.create_drg + create_drg = var.enable_drg && var.create_drg create_drg_attachment = var.enable_drg && var.create_drg_attachment && var.create_vcn - drg_id = var.create_drg ? try(oci_core_drg.vcn_drg.0.id, null) : var.drg_id + drg_id = var.create_drg ? try(oci_core_drg.vcn_drg.0.id, null) : var.drg_id - tcp_protocol = "6" - icmp_protocol = "1" - udp_protocol = "17" + tcp_protocol = "6" + icmp_protocol = "1" + udp_protocol = "17" service_cidr_block = lookup(data.oci_core_services.all_oci_services.services[0], "cidr_block") } diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/routing.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/routing.tf index 29e337319..c68a19823 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/routing.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/routing.tf @@ -1,32 +1,32 @@ resource "oci_core_route_table" "bastion_route_table" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.bastion_subnet_name + display_name = var.bastion_subnet_name dynamic "route_rules" { for_each = var.bastion_subnet_private ? [0] : [] content { network_entity_id = local.service_gateway_id - destination_type = "SERVICE_CIDR_BLOCK" - destination = local.service_cidr_block - description = "Route for all internal OCI services in the region" + destination_type = "SERVICE_CIDR_BLOCK" + destination = local.service_cidr_block + description = "Route for all internal OCI services in the region" } } dynamic "route_rules" { for_each = var.bastion_subnet_private ? [] : [0] content { network_entity_id = oci_core_internet_gateway.internet_gateway[0].id - destination_type = "CIDR_BLOCK" - destination = "0.0.0.0/0" - description = "Route to reach external Internet through the Internet gateway" + destination_type = "CIDR_BLOCK" + destination = "0.0.0.0/0" + description = "Route to reach external Internet through the Internet gateway" } } dynamic "route_rules" { for_each = var.bastion_subnet_private ? [0] : [] content { network_entity_id = local.nat_gateway_id - destination_type = "CIDR_BLOCK" - destination = "0.0.0.0/0" - description = "Route to reach external Internet through a NAT gateway" + destination_type = "CIDR_BLOCK" + destination = "0.0.0.0/0" + description = "Route to reach external Internet through a NAT gateway" } } dynamic "route_rules" { @@ -44,32 +44,32 @@ resource "oci_core_route_table" "bastion_route_table" { resource "oci_core_route_table" "cp_route_table" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.cp_subnet_name + display_name = var.cp_subnet_name dynamic "route_rules" { for_each = var.cp_subnet_private ? [0] : [] content { network_entity_id = local.service_gateway_id - destination_type = "SERVICE_CIDR_BLOCK" - destination = local.service_cidr_block - description = "Route for all internal OCI services in the region" + destination_type = "SERVICE_CIDR_BLOCK" + destination = local.service_cidr_block + description = "Route for all internal OCI services in the region" } } dynamic "route_rules" { for_each = var.cp_subnet_private ? [] : [0] content { network_entity_id = oci_core_internet_gateway.internet_gateway[0].id - destination_type = "CIDR_BLOCK" - destination = "0.0.0.0/0" - description = "Route to reach external Internet through the Internet gateway" + destination_type = "CIDR_BLOCK" + destination = "0.0.0.0/0" + description = "Route to reach external Internet through the Internet gateway" } } dynamic "route_rules" { for_each = local.cp_nat_mode ? [0] : [] content { network_entity_id = local.nat_gateway_id - destination_type = "CIDR_BLOCK" - destination = "0.0.0.0/0" - description = "Route to reach external Internet through a NAT gateway" + destination_type = "CIDR_BLOCK" + destination = "0.0.0.0/0" + description = "Route to reach external Internet through a NAT gateway" } } dynamic "route_rules" { @@ -87,12 +87,12 @@ resource "oci_core_route_table" "cp_route_table" { resource "oci_core_route_table" "lb_ext_route_table" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.external_lb_subnet_name + display_name = var.external_lb_subnet_name route_rules { network_entity_id = oci_core_internet_gateway.internet_gateway[0].id - destination_type = "CIDR_BLOCK" - destination = "0.0.0.0/0" - description = "Route to reach external Internet through the Internet gateway" + destination_type = "CIDR_BLOCK" + destination = "0.0.0.0/0" + description = "Route to reach external Internet through the Internet gateway" } dynamic "route_rules" { for_each = var.enable_drg ? var.peer_vcns : [] @@ -109,12 +109,12 @@ resource "oci_core_route_table" "lb_ext_route_table" { resource "oci_core_route_table" "lb_int_route_table" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.internal_lb_subnet_name + display_name = var.internal_lb_subnet_name route_rules { network_entity_id = local.service_gateway_id - destination_type = "SERVICE_CIDR_BLOCK" - destination = local.service_cidr_block - description = "Route for all internal OCI services in the region" + destination_type = "SERVICE_CIDR_BLOCK" + destination = local.service_cidr_block + description = "Route for all internal OCI services in the region" } dynamic "route_rules" { for_each = var.enable_drg ? var.peer_vcns : [] @@ -131,18 +131,18 @@ resource "oci_core_route_table" "lb_int_route_table" { resource "oci_core_route_table" "worker_route_table" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.worker_subnet_name + display_name = var.worker_subnet_name route_rules { network_entity_id = local.service_gateway_id - destination_type = "SERVICE_CIDR_BLOCK" - destination = local.service_cidr_block - description = "Route for all internal OCI services in the region" + destination_type = "SERVICE_CIDR_BLOCK" + destination = local.service_cidr_block + description = "Route for all internal OCI services in the region" } route_rules { network_entity_id = local.nat_gateway_id - destination_type = "CIDR_BLOCK" - destination = "0.0.0.0/0" - description = "Route to reach external Internet through a NAT gateway" + destination_type = "CIDR_BLOCK" + destination = "0.0.0.0/0" + description = "Route to reach external Internet through a NAT gateway" } dynamic "route_rules" { for_each = var.enable_drg ? var.peer_vcns : [] @@ -159,18 +159,18 @@ resource "oci_core_route_table" "worker_route_table" { resource "oci_core_route_table" "pod_route_table" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.pod_subnet_name + display_name = var.pod_subnet_name route_rules { network_entity_id = local.service_gateway_id - destination_type = "SERVICE_CIDR_BLOCK" - destination = local.service_cidr_block - description = "Route for all internal OCI services in the region" + destination_type = "SERVICE_CIDR_BLOCK" + destination = local.service_cidr_block + description = "Route for all internal OCI services in the region" } route_rules { network_entity_id = local.nat_gateway_id - destination_type = "CIDR_BLOCK" - destination = "0.0.0.0/0" - description = "Route to reach external Internet through a NAT gateway" + destination_type = "CIDR_BLOCK" + destination = "0.0.0.0/0" + description = "Route to reach external Internet through a NAT gateway" } dynamic "route_rules" { for_each = var.enable_drg ? var.peer_vcns : [] @@ -187,12 +187,12 @@ resource "oci_core_route_table" "pod_route_table" { resource "oci_core_route_table" "fss_route_table" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.fss_subnet_name + display_name = var.fss_subnet_name route_rules { network_entity_id = local.service_gateway_id - destination_type = "SERVICE_CIDR_BLOCK" - destination = local.service_cidr_block - description = "Route for all internal OCI services in the region" + destination_type = "SERVICE_CIDR_BLOCK" + destination = local.service_cidr_block + description = "Route for all internal OCI services in the region" } count = local.create_fss_subnet ? 1 : 0 } \ No newline at end of file diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/security-list.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/security-list.tf index 824fb09d9..19cd1482c 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/security-list.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/security-list.tf @@ -1,7 +1,7 @@ resource "oci_core_security_list" "cp_sl" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.cp_subnet_name + display_name = var.cp_subnet_name # Ingress rules and their corresponding egress ingress_security_rules { @@ -22,10 +22,10 @@ resource "oci_core_security_list" "cp_sl" { code = "4" type = "3" } - protocol = local.icmp_protocol - destination = "0.0.0.0/0" + protocol = local.icmp_protocol + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } ingress_security_rules { @@ -44,10 +44,10 @@ resource "oci_core_security_list" "cp_sl" { icmp_options { type = "3" } - protocol = local.icmp_protocol - destination = oci_core_vcn.spoke_vcn[0].cidr_block + protocol = local.icmp_protocol + destination = oci_core_vcn.spoke_vcn[0].cidr_block destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } count = local.create_cp_subnet ? 1 : 0 @@ -56,7 +56,7 @@ resource "oci_core_security_list" "cp_sl" { resource "oci_core_security_list" "external_lb_sl" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.external_lb_subnet_name + display_name = var.external_lb_subnet_name # Ingress rules and their corresponding egress ingress_security_rules { @@ -77,10 +77,10 @@ resource "oci_core_security_list" "external_lb_sl" { code = "4" type = "3" } - protocol = local.icmp_protocol - destination = "0.0.0.0/0" + protocol = local.icmp_protocol + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } ingress_security_rules { @@ -99,10 +99,10 @@ resource "oci_core_security_list" "external_lb_sl" { icmp_options { type = "3" } - protocol = local.icmp_protocol - destination = oci_core_vcn.spoke_vcn[0].cidr_block + protocol = local.icmp_protocol + destination = oci_core_vcn.spoke_vcn[0].cidr_block destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } count = local.create_external_lb_subnet ? 1 : 0 @@ -113,7 +113,7 @@ resource "oci_core_security_list" "external_lb_sl" { resource "oci_core_security_list" "internal_lb_sl" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.internal_lb_subnet_name + display_name = var.internal_lb_subnet_name # Ingress rules and their corresponding egress ingress_security_rules { @@ -134,10 +134,10 @@ resource "oci_core_security_list" "internal_lb_sl" { code = "4" type = "3" } - protocol = local.icmp_protocol - destination = "0.0.0.0/0" + protocol = local.icmp_protocol + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } ingress_security_rules { @@ -156,10 +156,10 @@ resource "oci_core_security_list" "internal_lb_sl" { icmp_options { type = "3" } - protocol = local.icmp_protocol - destination = oci_core_vcn.spoke_vcn[0].cidr_block + protocol = local.icmp_protocol + destination = oci_core_vcn.spoke_vcn[0].cidr_block destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } count = local.create_internal_lb_subnet ? 1 : 0 @@ -169,7 +169,7 @@ resource "oci_core_security_list" "internal_lb_sl" { resource "oci_core_security_list" "worker_sl" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.worker_subnet_name + display_name = var.worker_subnet_name # Ingress rules and their corresponding egress ingress_security_rules { @@ -190,10 +190,10 @@ resource "oci_core_security_list" "worker_sl" { code = "4" type = "3" } - protocol = local.icmp_protocol - destination = "0.0.0.0/0" + protocol = local.icmp_protocol + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } ingress_security_rules { @@ -212,10 +212,10 @@ resource "oci_core_security_list" "worker_sl" { icmp_options { type = "3" } - protocol = local.icmp_protocol - destination = oci_core_vcn.spoke_vcn[0].cidr_block + protocol = local.icmp_protocol + destination = oci_core_vcn.spoke_vcn[0].cidr_block destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } count = local.create_worker_subnet ? 1 : 0 @@ -224,7 +224,7 @@ resource "oci_core_security_list" "worker_sl" { resource "oci_core_security_list" "pod_sl" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.pod_subnet_name + display_name = var.pod_subnet_name # Ingress rules and their corresponding egress ingress_security_rules { @@ -245,10 +245,10 @@ resource "oci_core_security_list" "pod_sl" { code = "4" type = "3" } - protocol = local.icmp_protocol - destination = "0.0.0.0/0" + protocol = local.icmp_protocol + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } ingress_security_rules { @@ -267,10 +267,10 @@ resource "oci_core_security_list" "pod_sl" { icmp_options { type = "3" } - protocol = local.icmp_protocol - destination = oci_core_vcn.spoke_vcn[0].cidr_block + protocol = local.icmp_protocol + destination = oci_core_vcn.spoke_vcn[0].cidr_block destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } count = local.create_pod_subnet ? 1 : 0 @@ -280,7 +280,7 @@ resource "oci_core_security_list" "pod_sl" { resource "oci_core_security_list" "fss_sl" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.fss_subnet_name + display_name = var.fss_subnet_name # Ingress rules and their corresponding egress ingress_security_rules { @@ -301,10 +301,10 @@ resource "oci_core_security_list" "fss_sl" { code = "4" type = "3" } - protocol = local.icmp_protocol - destination = "0.0.0.0/0" + protocol = local.icmp_protocol + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } ingress_security_rules { @@ -323,10 +323,10 @@ resource "oci_core_security_list" "fss_sl" { icmp_options { type = "3" } - protocol = local.icmp_protocol - destination = oci_core_vcn.spoke_vcn[0].cidr_block + protocol = local.icmp_protocol + destination = oci_core_vcn.spoke_vcn[0].cidr_block destination_type = "CIDR_BLOCK" - stateless = "true" + stateless = "true" } count = local.create_fss_subnet ? 1 : 0 @@ -335,14 +335,14 @@ resource "oci_core_security_list" "fss_sl" { resource "oci_core_security_list" "bastion_security_list" { compartment_id = var.network_compartment_id vcn_id = local.vcn_id - display_name = var.bastion_subnet_name + display_name = var.bastion_subnet_name # Ingress rules and their corresponding egress ingress_security_rules { - protocol = local.tcp_protocol + protocol = local.tcp_protocol source_type = "CIDR_BLOCK" - source = "0.0.0.0/0" - stateless = true + source = "0.0.0.0/0" + stateless = true description = "Allow SSH connections to the subnet. Can be deleted if only using OCI Bastion subnet" tcp_options { max = 22 @@ -351,11 +351,11 @@ resource "oci_core_security_list" "bastion_security_list" { } egress_security_rules { - destination = "0.0.0.0/0" + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - protocol = local.tcp_protocol - stateless = true - description = "Allow SSH responses from the subnet" + protocol = local.tcp_protocol + stateless = true + description = "Allow SSH responses from the subnet" tcp_options { source_port_range { max = 22 @@ -366,26 +366,26 @@ resource "oci_core_security_list" "bastion_security_list" { # Egress rules and their corresponding ingress egress_security_rules { - destination = var.vcn_cidr_blocks[0] + destination = var.vcn_cidr_blocks[0] destination_type = "CIDR_BLOCK" - protocol = "all" - stateless = true - description = "Enable the bastion hosts to reach the entire VCN" + protocol = "all" + stateless = true + description = "Enable the bastion hosts to reach the entire VCN" } egress_security_rules { - destination = "0.0.0.0/0" + destination = "0.0.0.0/0" destination_type = "CIDR_BLOCK" - protocol = "all" - stateless = false - description = "Let bastion instances connect to Internet" + protocol = "all" + stateless = false + description = "Let bastion instances connect to Internet" } ingress_security_rules { - protocol = "all" + protocol = "all" source_type = "CIDR_BLOCK" - source = var.vcn_cidr_blocks[0] - stateless = true + source = var.vcn_cidr_blocks[0] + stateless = true description = "Allow responses from the VCN to the bastion hosts" } diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/subnet.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/subnet.tf index cf8c69d3f..9f552bc52 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/subnet.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/subnet.tf @@ -1,91 +1,91 @@ resource "oci_core_subnet" "external_lb_subnet" { - cidr_block = var.external_lb_cidr - compartment_id = var.network_compartment_id - vcn_id = local.vcn_id + cidr_block = var.external_lb_cidr + compartment_id = var.network_compartment_id + vcn_id = local.vcn_id prohibit_public_ip_on_vnic = false - dns_label = var.external_lb_subnet_dns_label - display_name = var.external_lb_subnet_name - route_table_id = oci_core_route_table.lb_ext_route_table.0.id - security_list_ids = [oci_core_security_list.external_lb_sl.0.id] - dhcp_options_id = oci_core_dhcp_options.external_lb_dhcp[0].id - count = local.create_external_lb_subnet ? 1 : 0 + dns_label = var.external_lb_subnet_dns_label + display_name = var.external_lb_subnet_name + route_table_id = oci_core_route_table.lb_ext_route_table.0.id + security_list_ids = [oci_core_security_list.external_lb_sl.0.id] + dhcp_options_id = oci_core_dhcp_options.external_lb_dhcp[0].id + count = local.create_external_lb_subnet ? 1 : 0 } resource "oci_core_subnet" "internal_lb_subnet" { - cidr_block = var.internal_lb_cidr - compartment_id = var.network_compartment_id - vcn_id = local.vcn_id + cidr_block = var.internal_lb_cidr + compartment_id = var.network_compartment_id + vcn_id = local.vcn_id prohibit_public_ip_on_vnic = true - dns_label = var.internal_lb_subnet_dns_label - display_name = var.internal_lb_subnet_name - route_table_id = oci_core_route_table.lb_int_route_table.0.id - security_list_ids = [oci_core_security_list.internal_lb_sl.0.id] - dhcp_options_id = oci_core_dhcp_options.internal_lb_dhcp[0].id - count = local.create_internal_lb_subnet ? 1 : 0 + dns_label = var.internal_lb_subnet_dns_label + display_name = var.internal_lb_subnet_name + route_table_id = oci_core_route_table.lb_int_route_table.0.id + security_list_ids = [oci_core_security_list.internal_lb_sl.0.id] + dhcp_options_id = oci_core_dhcp_options.internal_lb_dhcp[0].id + count = local.create_internal_lb_subnet ? 1 : 0 } resource "oci_core_subnet" "oke_cp_subnet" { - cidr_block = var.cp_subnet_cidr - compartment_id = var.network_compartment_id - vcn_id = local.vcn_id - dns_label = var.cp_subnet_dns_label - display_name = var.cp_subnet_name + cidr_block = var.cp_subnet_cidr + compartment_id = var.network_compartment_id + vcn_id = local.vcn_id + dns_label = var.cp_subnet_dns_label + display_name = var.cp_subnet_name prohibit_public_ip_on_vnic = var.cp_subnet_private - route_table_id = oci_core_route_table.cp_route_table.0.id - security_list_ids = [oci_core_security_list.cp_sl.0.id] - dhcp_options_id = oci_core_dhcp_options.oke_cp_dhcp[0].id - count = local.create_cp_subnet ? 1 : 0 + route_table_id = oci_core_route_table.cp_route_table.0.id + security_list_ids = [oci_core_security_list.cp_sl.0.id] + dhcp_options_id = oci_core_dhcp_options.oke_cp_dhcp[0].id + count = local.create_cp_subnet ? 1 : 0 } resource "oci_core_subnet" "worker_subnet" { - cidr_block = var.worker_subnet_cidr - compartment_id = var.network_compartment_id - vcn_id = local.vcn_id - dns_label = var.worker_subnet_dns_label - display_name = var.worker_subnet_name + cidr_block = var.worker_subnet_cidr + compartment_id = var.network_compartment_id + vcn_id = local.vcn_id + dns_label = var.worker_subnet_dns_label + display_name = var.worker_subnet_name prohibit_public_ip_on_vnic = true - route_table_id = oci_core_route_table.worker_route_table.0.id - security_list_ids = [oci_core_security_list.worker_sl.0.id] - dhcp_options_id = oci_core_dhcp_options.worker_dhcp[0].id - count = local.create_worker_subnet ? 1 : 0 + route_table_id = oci_core_route_table.worker_route_table.0.id + security_list_ids = [oci_core_security_list.worker_sl.0.id] + dhcp_options_id = oci_core_dhcp_options.worker_dhcp[0].id + count = local.create_worker_subnet ? 1 : 0 } resource "oci_core_subnet" "pods_subnet" { - cidr_block = var.pod_subnet_cidr - compartment_id = var.network_compartment_id - vcn_id = local.vcn_id - dns_label = var.pod_subnet_dns_label - display_name = var.pod_subnet_name + cidr_block = var.pod_subnet_cidr + compartment_id = var.network_compartment_id + vcn_id = local.vcn_id + dns_label = var.pod_subnet_dns_label + display_name = var.pod_subnet_name prohibit_public_ip_on_vnic = true - route_table_id = oci_core_route_table.pod_route_table.0.id - security_list_ids = [oci_core_security_list.pod_sl.0.id] - dhcp_options_id = oci_core_dhcp_options.pods_dhcp[0].id - count = local.create_pod_subnet ? 1 : 0 + route_table_id = oci_core_route_table.pod_route_table.0.id + security_list_ids = [oci_core_security_list.pod_sl.0.id] + dhcp_options_id = oci_core_dhcp_options.pods_dhcp[0].id + count = local.create_pod_subnet ? 1 : 0 } resource "oci_core_subnet" "bastion_subnet" { - cidr_block = var.bastion_subnet_cidr - compartment_id = var.network_compartment_id - vcn_id = local.vcn_id - dns_label = var.bastion_subnet_dns_label - display_name = var.bastion_subnet_name + cidr_block = var.bastion_subnet_cidr + compartment_id = var.network_compartment_id + vcn_id = local.vcn_id + dns_label = var.bastion_subnet_dns_label + display_name = var.bastion_subnet_name prohibit_public_ip_on_vnic = var.bastion_subnet_private - route_table_id = oci_core_route_table.bastion_route_table.0.id - security_list_ids = [oci_core_security_list.bastion_security_list.0.id] - dhcp_options_id = oci_core_dhcp_options.bastion_dhcp[0].id - count = local.create_bastion_subnet? 1 : 0 + route_table_id = oci_core_route_table.bastion_route_table.0.id + security_list_ids = [oci_core_security_list.bastion_security_list.0.id] + dhcp_options_id = oci_core_dhcp_options.bastion_dhcp[0].id + count = local.create_bastion_subnet ? 1 : 0 } resource "oci_core_subnet" "fss_subnet" { - cidr_block = var.fss_subnet_cidr - compartment_id = var.network_compartment_id - vcn_id = local.vcn_id - dns_label = var.fss_subnet_dns_label - display_name = var.fss_subnet_name + cidr_block = var.fss_subnet_cidr + compartment_id = var.network_compartment_id + vcn_id = local.vcn_id + dns_label = var.fss_subnet_dns_label + display_name = var.fss_subnet_name prohibit_public_ip_on_vnic = true - route_table_id = oci_core_route_table.fss_route_table.0.id - security_list_ids = [oci_core_security_list.fss_sl.0.id] - dhcp_options_id = oci_core_dhcp_options.fss_dhcp[0].id - count = local.create_fss_subnet ? 1 : 0 + route_table_id = oci_core_route_table.fss_route_table.0.id + security_list_ids = [oci_core_security_list.fss_sl.0.id] + dhcp_options_id = oci_core_dhcp_options.fss_dhcp[0].id + count = local.create_fss_subnet ? 1 : 0 } diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/vcn.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/vcn.tf index b29a9533c..adb57c5f2 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/vcn.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/modules/network/vcn.tf @@ -1,9 +1,9 @@ resource "oci_core_vcn" "spoke_vcn" { compartment_id = var.network_compartment_id - display_name = var.vcn_name - cidr_blocks = var.vcn_cidr_blocks - dns_label = var.vcn_dns_label - count = var.create_vcn ? 1 : 0 + display_name = var.vcn_name + cidr_blocks = var.vcn_cidr_blocks + dns_label = var.vcn_dns_label + count = var.create_vcn ? 1 : 0 } resource "oci_core_default_security_list" "lockdown" { diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/provider.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/provider.tf index 9eda735c3..526ee3cb7 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/provider.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/provider.tf @@ -6,7 +6,7 @@ terraform { version = "7.27.0" } null = { - source = "hashicorp/null" + source = "hashicorp/null" version = "3.2.4" } } diff --git a/app-dev/devops-and-containers/oke/oke-rm/infra/variable.tf b/app-dev/devops-and-containers/oke/oke-rm/infra/variable.tf index 7eab22711..01f77c741 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/infra/variable.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/infra/variable.tf @@ -9,7 +9,7 @@ variable "cni_type" { # VCN variable "create_vcn" { - type = bool + type = bool default = true } @@ -22,7 +22,7 @@ variable "vcn_name" { } variable "vcn_cidr_block" { - type = string + type = string default = "10.0.0.0/16" } @@ -33,7 +33,7 @@ variable "vcn_dns_label" { # CP SUBNET variable "create_cp_subnet" { - type = bool + type = bool default = true } @@ -42,7 +42,7 @@ variable "cp_subnet_name" { } variable "cp_subnet_private" { - type = bool + type = bool default = true } @@ -53,7 +53,7 @@ variable "cp_allowed_source_cidr" { # WORKER SUBNET variable "create_worker_subnet" { - type = bool + type = bool default = true } @@ -64,7 +64,7 @@ variable "worker_subnet_name" { # POD SUBNET variable "create_pod_subnet" { - type = bool + type = bool default = true } @@ -75,7 +75,7 @@ variable "pod_subnet_name" { # LB SUBNETS variable "create_external_lb_subnet" { - type = bool + type = bool default = true } @@ -84,7 +84,7 @@ variable "external_lb_subnet_name" { } variable "create_internal_lb_subnet" { - type = bool + type = bool default = true } @@ -95,12 +95,12 @@ variable "internal_lb_subnet_name" { # BASTION SUBNET variable "create_bastion_subnet" { - type = bool + type = bool default = true } variable "bastion_subnet_private" { - type = bool + type = bool default = false } @@ -111,7 +111,7 @@ variable "bastion_subnet_name" { # FSS SUBNET variable "create_fss" { - type = bool + type = bool default = true } @@ -120,24 +120,24 @@ variable "fss_subnet_name" { } variable "create_gateways" { - type = bool + type = bool default = true } variable "create_internet_gateway" { - type = bool + type = bool default = true } #CONTROL PLANE EXTERNAL CONNECTION variable "cp_external_nat" { - type = bool + type = bool default = true } variable "allow_external_cp_traffic" { - type = bool + type = bool default = true } @@ -148,12 +148,12 @@ variable "cp_egress_cidr" { # DRG variable "enable_drg" { - type = bool + type = bool default = false } variable "create_drg" { - type = bool + type = bool default = true } @@ -166,12 +166,12 @@ variable "drg_name" { } variable "create_drg_attachment" { - type = bool + type = bool default = true } variable "peer_vcns" { - type = list(string) + type = list(string) default = [] } diff --git a/app-dev/devops-and-containers/oke/oke-rm/ingress.md b/app-dev/devops-and-containers/oke/oke-rm/ingress.md deleted file mode 100644 index 7d0d6c8a7..000000000 --- a/app-dev/devops-and-containers/oke/oke-rm/ingress.md +++ /dev/null @@ -1,432 +0,0 @@ -# Ingress guide and best practices in OKE - -Configuring an Ingress is one of the first steps any engineer needs to understand when dealing with Kubernetes. - -A full list of all the annotations both for Load Balancers and Network Load Balancers can be found [here](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengcreatingloadbalancer_topic-Summaryofannotations.htm). - -There are some best practices and common tasks to perform, and as an example we will use Traefik OSS Helm chart, but you are free to use whatever ingress you like. - -## Prerequisites: policies - -Some policies are needed for the OKE cluster to interact with NSGs during this guide, so better have everything already in place! -``` -ALLOW any-user to manage network-security-groups in compartment where request.principal.type = 'cluster' -ALLOW any-user to manage vcns in compartment where request.principal.type = 'cluster' -ALLOW any-user to manage virtual-network-family in compartment where request.principal.type = 'cluster' -``` -Note that these policies are necessary even if the network compartment and the cluster compartment are the same! - -These policies can also be further restricted by specifying the OKE cluster id. - -## Configuring the Service of type LoadBalancer - -This OKE stack has created a NSG for all the Load Balancers called **oke-lb-nsg**. This is part of the infrastructure and should not be modified, as it contains the -security rules to connect the LB to the worker nodes. - -To adopt a GitOps approach, the LB created by OKE is configured through annotations. The list of all valid annotations can be found [here](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengcreatingloadbalancer_topic-Summaryofannotations.htm). - -As a starting point, I would suggest these annotations, to be configured directly in the Helm chart: -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "None" -``` -These annotations will create a flexible public Load Balancer and attach the right NSG. Note that we have put the security-rule management-mode to None, and we will come back to that one later. - - - -### Private Load Balancer - -If you have chosen to create the Load Balancer in a private subnet, you will need an additional annotation, or the creation will fail: -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "None" - service.beta.kubernetes.io/oci-load-balancer-internal: "true" -``` -**service.beta.kubernetes.io/oci-load-balancer-internal** is required to create a LB in a private subnet. - -## Specifying externalTrafficPolicy to Local - -By default, many Ingress controllers are exposed through a Service with externalTrafficPolicy set to Cluster by default. - -This means that the Load Balancer may potentially send traffic to a node where no Ingress controller pod is scheduled. To avoid this, it is a best practice to set externalTrafficPolicy to Local: - -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "None" - spec: - externalTrafficPolicy: "Local" -``` - -NOTE: as a side effect, you will see in the OCI Load Balancer that not all the backend nodes report a successful health check. This is normal, as only nodes with an Ingress pod scheduled will actually receive traffic and respond to the health check. - -## Assigning a Public Reserved IP to the Load Balancer - -If you want to use a public reserved IP for the Load Balancer, you need to specify it in the Service spec: - -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "None" - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" # Your public reserved IP -``` -NOTE: If the public reserved IP is in a different compartment that the OKE cluster, you will need an additional policy: -```text -ALLOW any-user to read public-ips in tenancy where request.principal.type = 'cluster' -ALLOW any-user to manage floating-ips in tenancy where request.principal.type = 'cluster' -``` - -## Configuring LB ingress access from OKE - -The main idea here is to have a separate NSG managed by OKE in which the ingress security rules are defined. -We can accomplish this by specifying the NSG mode in **security-rule-management-mode** and by using **loadBalancerSourceRanges** in the Service specification: -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` -OKE will create a **frontend** NSG and attach it directly to the Load Balancer. It will allow all traffic coming from the CIDR blocks specified in **loadBalancerSourceRanges** on the ports exposed by the Service. -You are now capable of controlling the Ingress CIDR block allow rules declaratively! - -## Redirect HTTP traffic to HTTPS - -Often, the Service used by Ingress controllers exposes both port 80 and 443. It is then common to configure redirection of HTTP traffic to HTTPS. -While this feature is something that is supported in the OCI Load Balancer, there is no annotation to configure it from OKE. - -It is often preferable and easier to implement it at the Ingress level. -For example, for Traefik you just need to specify this in the Helm chart values: - -```yaml -ports: - web: - redirections: - to: websecure - scheme: https - websecure: - asDefault: true -``` - -## Ensure High Availability for ingress pods - -Ensuring that ingress pods are replicated and spread across different nodes is critical to ensure high availability. This is because ingress pods will handle all the ingress traffic -to the cluster. - -Be sure to have at least 3 ingress pods and to configure a **HorizontalPodAutoscaler** to dynamically scale replicas. - -To ensure distribution of ingress pods across AD/FD and nodes, it's better to define **TopologySpreadConstrain** and **pod anti-affinity**: -```yaml -topologySpreadConstraints: - - maxSkew: 1 - topologyKey: "topology.kubernetes.io/zone" # Distribute pods across ADs, put "oci.oraclecloud.com/fault-domain" if you have a region with only 1 AD, so that pods are distributed across FDs - whenUnsatisfiable: "ScheduleAnyway" # Soft rule - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ template "traefik.name" . }}' - -affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: # Soft rule - - podAffinityTerm: - labelSelector: - matchLabels: - app.kubernetes.io/name: '{{ template "traefik.name" . }}' - topologyKey: "kubernetes.io/hostname" # Try to distribute pods in different nodes. Better keep this rule as soft, otherwise max number of pods == number of current nodes - weight: 100 -``` - -## Use proxy protocol v2 to keep caller IP address - -Often, the security team will require the Kubernetes administrator to keep the source IP address of the caller in the X-Forwarded-For header, so that it is logged in the -Ingress Controller access logs. - -The cloud native environment is also full of reverse proxy, and so it is difficult to keep the original source IP. Thankfully, Proxy Protocol solves this, but it has to -be enabled both at the Load Balancer level and at the Ingress pod level: - -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - service.beta.kubernetes.io/oci-load-balancer-backend-protocol: "TCP" # Proxy Protocol only works with a TCP listener - service.beta.kubernetes.io/oci-load-balancer-connection-proxy-protocol-version: "2" # Enable Proxy Protocol v2 - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` - -Although we have enabled Proxy Protocol v2 at the Load Balancer level, the same must be done for the Ingress. - -For Traefik, proxy protocol is enabled by default, but Traefik implements a mechanism that requires the administrator to trust the IP ranges of the incoming requests, that is, -no X-Forwarded-For header will be populated unless the sender proxy IP is not explicitly trusted. -Let's then trust all the IP addresses of the VCN where the cluster is installed: -```yaml -ports: - web: - redirections: - to: websecure - scheme: https - websecure: - asDefault: true - forwardedHeaders: - trustedIPs: - - "10.0.0.0/16" - proxyProtocol: - trustedIPs: - - "10.0.0.0/16" -``` -Where 10.0.0.0/16 is the CIDR block of the VCN where the OKE cluster has been provisioned and where the Load Balancer is located. - -## Select only the necessary worker nodes to be included in the Load Balancer - -By default, OKE will include all the worker nodes in a cluster as backend set of the Load Balancer. If nodes increase a lot, having many nodes in the backend set -may slow down the Load Balancer. -We can restrict the nodes to be included in the backend set by using labels and the annotation **oci.oraclecloud.com/node-label-selector**: -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - service.beta.kubernetes.io/oci-load-balancer-backend-protocol: "TCP" # Proxy Protocol only works with a TCP listener - service.beta.kubernetes.io/oci-load-balancer-connection-proxy-protocol-version: "2" # Enable Proxy Protocol v2 - oci.oraclecloud.com/node-label-selector: "env=test" - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` -See the [documentation](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengconfiguringloadbalancersnetworkloadbalancers-subtopic.htm#contengcreatingloadbalancer_topic-Selecting_worker_nodes_to_include_in_backend_sets) for more examples. - -## Change the default Load Balancer policy if needed - -The default Load Balancer policy is ROUND_ROBIN. If your applications require long connection times, better change the policy to LEAST_CONNECTIONS: -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - service.beta.kubernetes.io/oci-load-balancer-backend-protocol: "TCP" # Proxy Protocol only works with a TCP listener - service.beta.kubernetes.io/oci-load-balancer-connection-proxy-protocol-version: "2" # Enable Proxy Protocol v2 - oci.oraclecloud.com/node-label-selector: "env=test" - oci.oraclecloud.com/loadbalancer-policy: "LEAST_CONNECTIONS" - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` - -## Change the default Connection Idle timeout - -By default, the Load Balancer TCP listener will keep a session alive even if there are no request/response interactions for 5 minutes. -Depending on your requirements, you have the possibility to reduce this idle time. Here, I am setting it to last 60 seconds: - -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - service.beta.kubernetes.io/oci-load-balancer-backend-protocol: "TCP" # Proxy Protocol only works with a TCP listener - service.beta.kubernetes.io/oci-load-balancer-connection-proxy-protocol-version: "2" # Enable Proxy Protocol v2 - oci.oraclecloud.com/node-label-selector: "env=test" - service.beta.kubernetes.io/oci-load-balancer-connection-idle-timeout: "60" - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` - -## Change the default health check parameters - -By default, health check on nodes will be performed by the Load Balancer every 10 seconds. Although Kubernetes will still forward traffic to different pods on different nodes in case of disruption, -it is safer to set it to a lower level: - -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - service.beta.kubernetes.io/oci-load-balancer-backend-protocol: "TCP" # Proxy Protocol only works with a TCP listener - service.beta.kubernetes.io/oci-load-balancer-connection-proxy-protocol-version: "2" # Enable Proxy Protocol v2 - oci.oraclecloud.com/node-label-selector: "env=test" - service.beta.kubernetes.io/oci-load-balancer-connection-idle-timeout: "60" - service.beta.kubernetes.io/oci-load-balancer-health-check-interval: "3000" - service.beta.kubernetes.io/oci-load-balancer-health-check-timeout: "2000" - service.beta.kubernetes.io/oci-load-balancer-health-check-retries: "3" - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` - -## Restrict HTTP header size - -As a security measure, it is better to restrict the HTTP header size. Here, I am restricting it to 16 KB: - -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - service.beta.kubernetes.io/oci-load-balancer-backend-protocol: "TCP" # Proxy Protocol only works with a TCP listener - service.beta.kubernetes.io/oci-load-balancer-connection-proxy-protocol-version: "2" # Enable Proxy Protocol v2 - oci.oraclecloud.com/node-label-selector: "env=test" - service.beta.kubernetes.io/oci-load-balancer-connection-idle-timeout: "60" - service.beta.kubernetes.io/oci-load-balancer-health-check-interval: "3000" - service.beta.kubernetes.io/oci-load-balancer-health-check-timeout: "2000" - service.beta.kubernetes.io/oci-load-balancer-health-check-retries: "3" - oci.oraclecloud.com/oci-load-balancer-rule-sets: | - { - "header_size": { - "items": [ - { - "action": "HTTP_HEADER", - "httpLargeHeaderSizeInKB": 16 - } - ] - } - } - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` - -## Provision the Load Balancer in a different subnet - -You can specify a different subnet where to provision the OCI Load Balancer. This is very useful for hub/spoke architectures. - -```yaml -service: - type: LoadBalancer - annotations: - oci.oraclecloud.com/load-balancer-type: "lb" - service.beta.kubernetes.io/oci-load-balancer-shape: "flexible" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-min: "10" - service.beta.kubernetes.io/oci-load-balancer-shape-flex-max: "100" - oci.oraclecloud.com/oci-network-security-groups: "ocid1.networksecuritygroup.oc1...." # It is the oke-lb-nsg OCID - oci.oraclecloud.com/security-rule-management-mode: "NSG" - service.beta.kubernetes.io/oci-load-balancer-backend-protocol: "TCP" # Proxy Protocol only works with a TCP listener - service.beta.kubernetes.io/oci-load-balancer-connection-proxy-protocol-version: "2" # Enable Proxy Protocol v2 - oci.oraclecloud.com/node-label-selector: "env=test" - service.beta.kubernetes.io/oci-load-balancer-connection-idle-timeout: "60" - service.beta.kubernetes.io/oci-load-balancer-health-check-interval: "3000" - service.beta.kubernetes.io/oci-load-balancer-health-check-timeout: "2000" - service.beta.kubernetes.io/oci-load-balancer-health-check-retries: "3" - oci.oraclecloud.com/oci-load-balancer-rule-sets: | - { - "header_size": { - "items": [ - { - "action": "HTTP_HEADER", - "httpLargeHeaderSizeInKB": 16 - } - ] - } - } - service.beta.kubernetes.io/oci-load-balancer-subnet1: "ocid1.subnet.oc1...." - spec: - externalTrafficPolicy: "Local" - loadBalancerIP: "121.127.6.12" - loadBalancerSourceRanges: - - "10.1.0.0/16" -``` - -## Additional best practices - -If you expect to have multiple environments in the same OKE cluster, it's better to create multiple IngressClasses for every environment, each with its own ingress controller and Load Balancer. - -To better manage costs, do not forget to add cost-tracking tags to the Load Balancer! See [here](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengtaggingclusterresources_tagging-oke-resources_load-balancer-tags.htm#contengtaggingclusterresources_tagging_oke_resources_load_balancer_tags) for more information. - -NOTE: Remember that to apply tags additional policies may be needed, see [here](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengtaggingclusterresources_iam-tag-namespace-policy.htm#contengtaggingclusterresources_iam-tag-namespace-policy). - -This guide shows how to configure an ingress controller with a Load Balancer configured with TLS passthrough. SSL/TLS termination will happen at the Ingress level. - -Usually, this is preferable as the Ingress controller is directly integrated with cert-manager and is capable to handle multiple certificates. - -If you only have one certificate, you can also terminate TLS at the Load Balancer level and there are some additional [annotations](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengcreatingloadbalancers-subtopic.htm#creatinglbhttps). - -## Enable API Gateway features (requires an enterprise license) - -Generally speaking, all the major Ingress controllers (Nginx, Traefik, Kong) are open source, but many useful features require an enterprise license. - -If you are serious about developing in Kubernetes, it is suggested to have one to better manage and secure APIs. - -For example, with an enterprise license it is possible to integrate OIDC with the Ingress, so that developers do not need to deal with security in their applications. -It is also possible to establish some rate limiting for APIs and some controllers even offer the possibility to create developer portals! - -One of such example for OKE is Traefik, as it is nicely [integrated with OCI](https://traefik.io/solutions/oracle-and-traefik/). \ No newline at end of file diff --git a/app-dev/devops-and-containers/oke/oke-rm/oke/addons.tf b/app-dev/devops-and-containers/oke/oke-rm/oke/addons.tf index f2e84a3d1..3e6ce92de 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/oke/addons.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/oke/addons.tf @@ -92,8 +92,8 @@ resource "oci_containerengine_addon" "oke_cert_manager" { addon_name = "CertManager" cluster_id = module.oke.cluster_id remove_addon_resources_on_delete = true - depends_on = [module.oke] - count = local.enable_cert_manager ? 1 : 0 + depends_on = [module.oke] + count = local.enable_cert_manager ? 1 : 0 } resource "oci_containerengine_addon" "oke_metrics_server" { @@ -103,41 +103,41 @@ resource "oci_containerengine_addon" "oke_metrics_server" { dynamic "configurations" { for_each = local.metrics_server_addon_configs content { - key = configurations.key + key = configurations.key value = configurations.value } } depends_on = [module.oke, oci_containerengine_addon.oke_cert_manager] - count = local.enable_metrics_server ? 1 : 0 + count = local.enable_metrics_server ? 1 : 0 } resource "oci_containerengine_addon" "oke_coredns" { addon_name = "CoreDNS" cluster_id = module.oke.cluster_id remove_addon_resources_on_delete = false - override_existing = true + override_existing = true dynamic "configurations" { for_each = local.coredns_addon_configs content { - key = configurations.key + key = configurations.key value = configurations.value } } depends_on = [module.oke] - count = var.cluster_type == "enhanced" && local.override_coredns ? 1 : 0 + count = var.cluster_type == "enhanced" && local.override_coredns ? 1 : 0 } resource "oci_containerengine_addon" "oke_cluster_autoscaler" { - addon_name = "ClusterAutoscaler" - cluster_id = module.oke.cluster_id + addon_name = "ClusterAutoscaler" + cluster_id = module.oke.cluster_id remove_addon_resources_on_delete = true dynamic "configurations" { for_each = local.cluster_autoscaler_addon_configs content { - key = configurations.key + key = configurations.key value = configurations.value } } depends_on = [module.oke] - count = local.enable_cluster_autoscaler ? 1 : 0 + count = local.enable_cluster_autoscaler ? 1 : 0 } \ No newline at end of file diff --git a/app-dev/devops-and-containers/oke/oke-rm/oke/locals.tf b/app-dev/devops-and-containers/oke/oke-rm/oke/locals.tf index 1c107893d..212f7b297 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/oke/locals.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/oke/locals.tf @@ -1,25 +1,25 @@ locals { - is_cp_subnet_private = data.oci_core_subnet.cp_subnet_data.prohibit_public_ip_on_vnic - is_lb_subnet_private = data.oci_core_subnet.lb_subnet_data.prohibit_public_ip_on_vnic - cni = var.cni_type == "vcn_native" ? "npn" : var.cni_type - is_flannel = var.cni_type == "flannel" - enable_cert_manager = var.cluster_type == "enhanced" && var.enable_cert_manager - enable_metrics_server = var.cluster_type == "enhanced" && var.enable_cert_manager && var.enable_metrics_server - enable_cluster_autoscaler = var.cluster_type == "enhanced" && var.enable_cluster_autoscaler + is_cp_subnet_private = data.oci_core_subnet.cp_subnet_data.prohibit_public_ip_on_vnic + is_lb_subnet_private = data.oci_core_subnet.lb_subnet_data.prohibit_public_ip_on_vnic + cni = var.cni_type == "vcn_native" ? "npn" : var.cni_type + is_flannel = var.cni_type == "flannel" + enable_cert_manager = var.cluster_type == "enhanced" && var.enable_cert_manager + enable_metrics_server = var.cluster_type == "enhanced" && var.enable_cert_manager && var.enable_metrics_server + enable_cluster_autoscaler = var.cluster_type == "enhanced" && var.enable_cluster_autoscaler create_autoscaler_policies = var.cluster_type == "enhanced" && var.enable_cluster_autoscaler && var.create_autoscaler_policies } # OIDC locals { - oidc_discovery_enabled = var.cluster_type == "enhanced" && var.enable_oidc_discovery + oidc_discovery_enabled = var.cluster_type == "enhanced" && var.enable_oidc_discovery oidc_authentication_enabled = var.cluster_type == "enhanced" && var.enable_oidc_authentication oidc_token_authentication_config = { - client_id = var.oidc_client_id - issuer_url = var.oidc_issuer - username_claim = var.oidc_username_claim + client_id = var.oidc_client_id + issuer_url = var.oidc_issuer + username_claim = var.oidc_username_claim username_prefix = var.oidc_username_prefix - groups_claim = var.oidc_groups_claim - groups_prefix = var.oidc_groups_prefix + groups_claim = var.oidc_groups_claim + groups_prefix = var.oidc_groups_prefix } } \ No newline at end of file diff --git a/app-dev/devops-and-containers/oke/oke-rm/oke/oke.tf b/app-dev/devops-and-containers/oke/oke-rm/oke/oke.tf index 33c5ff456..4491b2f50 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/oke/oke.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/oke/oke.tf @@ -2,76 +2,63 @@ # https://github.com/oracle-devrel/technology-engineering/blob/main/app-dev/devops-and-containers/oke/oke-policies/policies.md locals { - volume_kms_key_id = "" # kms OCID of the key used for in-transit and at-rest encryption of block volumes - ssh_public_key = "" # Insert the ssh public key to access worker nodes - - runcmd_growfs_oracle_linux = "sudo /usr/libexec/oci-growfs -y" - # Kubelet extra args is mostly used to taint nodes at start up in a node pool - kubelet_extra_args = "--register-with-taints=env=dev:NoSchedule,cluster=oke:PreferNoSchedule" + volume_kms_key_id = "" # kms OCID of the key used for in-transit and at-rest encryption of block volumes + ssh_public_key = "" # Insert the ssh public key to access worker nodes # Cloud init to taint nodes using Oracle Linux nodes. Make sure to disable the default cloud init - cloud_init_with_taint_ol = { + cloud_init_example = { runcmd = [ - "curl --fail -H \"Authorization: Bearer Oracle\" -L0 http://169.254.169.254/opc/v2/instance/metadata/oke_init_script | base64 --decode >/var/run/oke-init.sh", - local.runcmd_growfs_oracle_linux, - "bash /var/run/oke-init.sh --kubelet-extra-args \"${local.kubelet_extra_args}\"" + "echo \"example cloud init\"" ] } } module "oke" { - source = "oracle-terraform-modules/oke/oci" - version = "5.3.3" + source = "oracle-terraform-modules/oke/oci" + version = "5.3.3" compartment_id = var.oke_compartment_id - # IAM - Policies - create_iam_autoscaler_policy = "never" - create_iam_kms_policy = "never" - create_iam_operator_policy = "never" - create_iam_worker_policy = "never" # Network module - VCN + create_vcn = false + vcn_id = var.vcn_id + network_compartment_id = var.network_compartment_id + assign_public_ip_to_control_plane = !local.is_cp_subnet_private subnets = { - bastion = { create = "never"} + bastion = { create = "never" } operator = { create = "never" } - pub_lb = { id = local.is_lb_subnet_private ? null : var.lb_subnet_id } - int_lb = { id = local.is_lb_subnet_private ? var.lb_subnet_id : null } - cp = { id = var.cp_subnet_id } - workers = { id = var.worker_subnet_id } - pods = { id = local.is_flannel ? null : var.pod_subnet_id } + pub_lb = { id = local.is_lb_subnet_private ? null : var.lb_subnet_id } + int_lb = { id = local.is_lb_subnet_private ? var.lb_subnet_id : null } + cp = { id = var.cp_subnet_id } + workers = { id = var.worker_subnet_id } + pods = { id = local.is_flannel ? null : var.pod_subnet_id } } nsgs = { - bastion = { create = "never" } + bastion = { create = "never" } operator = { create = "never" } - pub_lb = {create = "never" } - int_lb = { create = "never"} - cp = { id = var.cp_nsg_id } - workers = { id = var.worker_nsg_id } - pods = { create = "never", id = var.cni_type == "flannel" ? null : var.pod_nsg_id } + pub_lb = { create = "never" } + int_lb = { create = "never" } + cp = { id = var.cp_nsg_id } + workers = { id = var.worker_nsg_id } + pods = { create = "never", id = var.cni_type == "flannel" ? null : var.pod_nsg_id } } - network_compartment_id = var.network_compartment_id - assign_public_ip_to_control_plane = ! local.is_cp_subnet_private - create_vcn = false - vcn_id = var.vcn_id # Network module - security control_plane_allowed_cidrs = var.cp_allowed_cidr_list - control_plane_is_public = ! local.is_cp_subnet_private - load_balancers = local.is_lb_subnet_private ? "internal" : "public" - preferred_load_balancer = local.is_lb_subnet_private ? "internal" : "public" + control_plane_is_public = !local.is_cp_subnet_private + load_balancers = local.is_lb_subnet_private ? "internal" : "public" + preferred_load_balancer = local.is_lb_subnet_private ? "internal" : "public" # Cluster module - create_cluster = true + create_cluster = true cluster_kms_key_id = var.cluster_kms_key_id - cluster_name = var.cluster_name - cluster_type = var.cluster_type - cni_type = local.cni + cluster_name = var.cluster_name + cluster_type = var.cluster_type + cni_type = local.cni kubernetes_version = var.kubernetes_version services_cidr = var.services_cidr - pods_cidr = var.pods_cidr - use_signed_images = false - use_defined_tags = false + pods_cidr = var.pods_cidr # OIDC - oidc_discovery_enabled = local.oidc_discovery_enabled - oidc_token_auth_enabled = local.oidc_authentication_enabled + oidc_discovery_enabled = local.oidc_discovery_enabled + oidc_token_auth_enabled = local.oidc_authentication_enabled oidc_token_authentication_config = local.oidc_token_authentication_config cluster_freeform_tags = { @@ -80,22 +67,20 @@ module "oke" { # Bastion create_bastion = false - # Operator create_operator = false - # OKE DATA PLANE (to configure) - + ########################## OKE DATA PLANE (to configure) ############################## # These are global configurations valid for all the node pools declared. You can see that the prefix is "worker_" because they apply to all workers of the cluster # You can override these global configurations in the node pool definition, and it will have precedence over the global ones. - worker_pool_mode = "node-pool" # Default mode should be node-pool for managed nodes, other modes are available for self-managed nodes, like instance and instance-pool, but be careful to have the required policy: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengdynamicgrouppolicyforselfmanagednodes.htm - worker_is_public = false # Workers should never be allowed to have a public ip + worker_pool_mode = "node-pool" # Default mode should be node-pool for managed nodes, other modes are available for self-managed nodes, like instance and instance-pool, but be careful to have the required policy: https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengdynamicgrouppolicyforselfmanagednodes.htm #ssh_public_key = local.ssh_public_key # De-comment if you want a ssh key to access the worker nodes, be sure to set the local variable - worker_image_type = "oke" # NOTE: the oke mode will fetch the latest OKE Oracle Linux image released by the OKE team. If you want more control, better to use "custom" and specify the image id. This is because an image id is always fixed, and controlled by you. + worker_image_type = "oke" # NOTE: the oke mode will fetch the latest OKE Oracle Linux image released by the OKE team. If you want more control, better to use "custom" and specify the image id. This is because an image id is always fixed, and controlled by you. #worker_image_id = "" # The image id to use for the worker nodes. For Oracle Linux images, check this link: https://docs.oracle.com/en-us/iaas/images/oke-worker-node-oracle-linux-8x/index.htm - # For Ubuntu images, you need to create an Ubuntu custom image in your tenancy first, and then set the OCID of the custom image here + # For Ubuntu images, you need to create an Ubuntu custom image in your tenancy first, and then set the OCID of the custom image here + # NOTE: set worker_image_type to "custom" and specify an image id to use custom images for all workers # Set this to true to enable in-transit encryption on all node pools by default # NOTE: in-transit encryption is supported only for paravirtualized attached block volumes and boot volumes, hence you will need to create another StorageClass in the cluster to attach volume through paravirtualization, as the default oci-bv StorageClass uses iSCSI @@ -103,111 +88,134 @@ module "oke" { worker_pv_transit_encryption = true # Enable encryption of volumes with a key managed by you, in your OCI Vault - #worker_volume_kms_key_id = local.volume_kms_key_id - # When using OCI_VCN_NATIVE_CNI, set the maximum number of pods for all nodes, must be between 1 and 110 - - #max_pods_per_node = 31 - - #worker_cloud_init = [{ content_type = "text/cloud-config", content = yamlencode(local.cloud_init_with_taint_ol)}] # Cloud init to add to all node pools. This will be added to the default_cloud_init - /* ABOUT CLOUD INIT The OKE module will automatically generate an optimal cloud-init for both Oracle Linux and Ubuntu nodes. This auto-generated cloud-init is called "default cloud-init". There is the possibility to disable this and to define your own cloud-init. This is not suggested unless you know what you are doing. For Oracle Linux, the oci-growfs command is already inserted in the default cloud-init. */ + #worker_cloud_init = [{ content_type = "text/cloud-config", content = yamlencode(local.cloud_init_example)}] # Cloud init to add to all node pools. This will be added to the default_cloud_init # GLOBAL TAGS TO BE APPLIED ON ALL NODES # NOTE: tags will be applied to both the node pool and the nodes - -/*workers_freeform_tags = { + /*workers_freeform_tags = { "oke-cluster-name" = var.cluster_name } workers_defined_tags = {} */ # GLOBAL NODE POOL LABELS TO BE APPLIED ON ALL NODES (Kubernetes labels) - #worker_node_labels = {} + # Additional NSGs to ba attached to all pods + # pod_nsg_ids = [] + + # Additional NSGs to be attached to all workers + #worker_nsg_ids = [] + + # When using VCN_NATIVE_CNI, set the maximum number of pods for all nodes, must be between 1 and 110 + #max_pods_per_node = 31 + + # Disable IMDSv1 endpoints for self-managed nodes + worker_legacy_imds_endpoints_disabled = true + + # Disable IMDSv1 endpoints for managed nodes + worker_node_metadata = { + areLegacyImdsEndpointsDisabled : "true" + } + # This is a collection of example node pools that you can use with the OKE module. Set create = true to provision them worker_pools = { # ORACLE LINUX - MANAGED NODE POOL np-ad1 = { - shape = "VM.Standard.E4.Flex" - size = 1 - kubernetes_version = var.kubernetes_version # You can set this variable with a constant, so that control plane and data plane are upgraded separately - placement_ads = ["1"] # As best practice, one node pool should be associated only to one specific AD - ocpus = 1 # No need to specify ocpus and memory if you are not using a Flex shape - memory = 16 - node_cycling_enabled = false # Option to enable/disable node pool cycling through Terraform. Only works with Enhanced clusters! - node_cycling_max_surge = "50%" + shape = "VM.Standard.E4.Flex" + size = 1 + kubernetes_version = var.kubernetes_version # You can set this variable with a constant, so that control plane and data plane are upgraded separately + placement_ads = ["1"] # As best practice, one node pool should be associated only to one specific AD + ocpus = 1 # No need to specify ocpus and memory if you are not using a Flex shape + memory = 16 + node_cycling_enabled = false # Option to enable/disable node pool cycling through Terraform. Only works with Enhanced clusters! + node_cycling_max_surge = "50%" node_cycling_max_unavailable = "25%" - node_cycling_mode = ["boot_volume"] # Valid values are instance and boot_volume. Only works when (kubernetes_version, image_id, boot_volume_size, node_metadata, ssh_public_key, volume_kms_key_id) are modified. If you need to change something else, switch to "instance" - # NOTE: boot_volume mode seems to work only for Flannel clusters for now - boot_volume_size = 100 - freeform_tags = { # Nodes in the node pool will be tagged with these freeform tags - "oke-cluster-name" = var.cluster_name + node_cycling_mode = ["instace"] # Valid values are instance and boot_volume. The boot_volume mode only works when (kubernetes_version, image_id, boot_volume_size, node_metadata, ssh_public_key, volume_kms_key_id) are modified. + boot_volume_size = 50 + # max_pods_per_node = 10 # When using VCN_NATIVE CNI, configure maximum number of pods for each node in the node pool + create = false # Set it to true so that the node pool is created + } + + # VIRTUAL NODE POOL + oke-virtual = { + description = "OKE-managed Virtual Node Pool" + shape = "Pod.Standard.E4.Flex" + mode = "virtual-node-pool" + placement_ads = ["1"] + taints = { + virtual-node-workload = { + value = "true" + effect = "NoSchedule" + } } - # max_pods_per_node = 10 # When using VCN_NATIVE CNI, configure maximum number of pods for each node in the node pool - create = false # Set it to true so that the node pool is created + size = 1 + create = false } # UBUNTU - MANAGED NODE POOL np-ad1-ubuntu = { - shape = "VM.Standard.E4.Flex" - size = 1 + shape = "VM.Standard.E4.Flex" + size = 1 kubernetes_version = var.kubernetes_version - placement_ads = ["1"] - ocpus = 1 - memory = 16 + placement_ads = ["1"] + ocpus = 1 + memory = 16 # NOTE! The OKE module will automatically verify the image and install the OKE Ubuntu Node package. You just need to create a custom image based on Ubuntu 22.04 or 24.04. Ubuntu Minimal is recommended - image_type = "custom" - image_id = "ocid1.image.oc1..." # Put your custom Ubuntu image here - node_cycling_enabled = false - node_cycling_max_surge = "50%" + image_type = "custom" + image_id = "ocid1.image..." # Put your custom Ubuntu image here + node_cycling_enabled = false + node_cycling_max_surge = "50%" node_cycling_max_unavailable = "25%" - node_cycling_mode = ["boot_volume"] # NOTE! Make sure you create the original Ubuntu VM with a boot volume of size 50 (the default). Depending on the boot volume size of the original VM, the custom image will require that minimum storage boot_volume_size = 100 - create = false + create = false } - # ORACLE LINUX - MANAGED NODE POOL WITH TAINTS - np-ad1-taints = { # An example of a node pool using a custom cloud-init script to define taints at the node pool level - shape = "VM.Standard.E4.Flex" - size = 1 + np-ad1-taints = { # An example of a node pool using a custom cloud-init script to define taints at the node pool level + shape = "VM.Standard.E4.Flex" + size = 1 placement_ads = ["1"] - ocpus = 1 - memory = 16 - disable_default_cloud_init = true # If you want to configure some kubelet arguments, make sure to disable the default cloud-init as the taints are defined through kubelet extra arguments - cloud_init = [{ content_type = "text/cloud-config", content = yamlencode(local.cloud_init_with_taint_ol)}] - node_cycling_enabled = false - node_cycling_max_surge = "50%" + ocpus = 1 + memory = 16 + taints = { + my-taint = { + value = "true" + effect = "NoSchedule" + } + } + node_cycling_enabled = false + node_cycling_max_surge = "50%" node_cycling_max_unavailable = "25%" - node_cycling_mode = ["boot_volume"] - boot_volume_size = 100 - create = false + boot_volume_size = 100 + create = false } # ORACLE LINUX/UBUNTU - SELF-MANAGED NODE oke-instance = { - shape = "VM.Standard.E4.Flex" - mode = "instance" - description = "Self managed instance" - size = 1 - placement_ads = ["1"] - ocpus = 1 - memory = 16 + shape = "VM.Standard.E4.Flex" + mode = "instance" + kubernetes_version = "v1.32.1" + description = "Self managed instance" + size = 1 + placement_ads = ["1"] + ocpus = 1 + memory = 16 # ENABLE IT FOR UBUNTU NODES - #image_type = "custom" - #image_id = "ocid1.image.oc1..." + image_type = "custom" + image_id = "ocid1.image..." boot_volume_size = 100 # Self-managed node specific parameters - boot_volume_vpus_per_gb = 10 # 10: Balanced, 20: High, 30-120: Ultra High (requires multipath) + boot_volume_vpus_per_gb = 10 # 10: Balanced, 20: High, 30-120: Ultra High (requires multipath) # Burstable instance #burst = "BASELINE_1_2" # Valid values BASELINE_1_8,BASELINE_1_2, only for Flex shapes! @@ -231,31 +239,26 @@ module "oke" { "OS Management Service Agent" = "DISABLED" } } - create = false } - - # CLUSTER AUTOSCALER - + ### CLUSTER AUTOSCALER # ORACLE LINUX SYSTEM NODES - MANAGED NODE POOL np-system-ad1 = { - shape = "VM.Standard.E4.Flex" - size = 1 - placement_ads = ["1"] - ocpus = 1 - memory = 16 - node_cycling_enabled = false - node_cycling_max_surge = "50%" + shape = "VM.Standard.E4.Flex" + size = 1 + placement_ads = ["1"] + ocpus = 1 + memory = 16 + node_cycling_enabled = false + node_cycling_max_surge = "50%" node_cycling_max_unavailable = "25%" - node_cycling_mode = ["boot_volume"] node_labels = { role = "system" } create = false } - # ORACLE LINUX AUTOSCALED - MANAGED NODE POOL /* This is a sample pool where autoscaling is enabled, note the freeform tag REQUIREMENTS FOR ENABLING THE CLUSTER AUTOSCALER @@ -267,17 +270,16 @@ module "oke" { - NODE POOL IS A MANAGED TYPE, CLUSTER AUTOSCALER DOES NOT WORK WITH SELF-MANAGED WORKER POOLS! */ np-autoscaled-ad1 = { - shape = "VM.Standard.E4.Flex" - size = 0 - placement_ads = ["1"] - ocpus = 1 - memory = 16 - node_cycling_enabled = false - node_cycling_max_surge = "50%" + shape = "VM.Standard.E4.Flex" + size = 0 + placement_ads = ["1"] + ocpus = 1 + memory = 16 + node_cycling_enabled = false + node_cycling_max_surge = "50%" node_cycling_max_unavailable = "25%" - node_cycling_mode = ["boot_volume"] - boot_volume_size = 100 - ignore_initial_pool_size = true # If set to true, node pool size drift won't be accounted in Terraform, useful also if this pool is autoscaled by an external component (cluster-autoscaler) or manually by a user + boot_volume_size = 100 + ignore_initial_pool_size = true # If set to true, node pool size drift won't be accounted in Terraform, useful also if this pool is autoscaled by an external component (cluster-autoscaler) or manually by a user freeform_tags = { cluster_autoscaler = "enabled" } @@ -287,22 +289,21 @@ module "oke" { # ORACLE LINUX AUTOSCALED PREEMPTIBLE - MANAGED NODE POOL # Often, to save money it makes sense to provision preemptible instances, as autoscaled node pools are already very dynamic np-autoscaled-preemptible-ad1 = { - shape = "VM.Standard.E4.Flex" - size = 1 - placement_ads = ["1"] - ocpus = 1 - memory = 16 - node_cycling_enabled = false - node_cycling_max_surge = "50%" + shape = "VM.Standard.E4.Flex" + size = 1 + placement_ads = ["1"] + ocpus = 1 + memory = 16 + node_cycling_enabled = false + node_cycling_max_surge = "50%" node_cycling_max_unavailable = "25%" - node_cycling_mode = ["boot_volume"] - boot_volume_size = 100 - ignore_initial_pool_size = true # If set to true, node pool size drift won't be accounted in Terraform, useful also if this pool is autoscaled by an external component (cluster-autoscaler) or manually by a user + boot_volume_size = 100 + ignore_initial_pool_size = true # If set to true, node pool size drift won't be accounted in Terraform, useful also if this pool is autoscaled by an external component (cluster-autoscaler) or manually by a user freeform_tags = { cluster_autoscaler = "enabled" } preemptible_config = { - enable = true + enable = true is_preserve_boot_volume = false } create = false diff --git a/app-dev/devops-and-containers/oke/oke-rm/oke/oke.zip b/app-dev/devops-and-containers/oke/oke-rm/oke/oke.zip index 87d4168b8..e80d8c4b3 100644 Binary files a/app-dev/devops-and-containers/oke/oke-rm/oke/oke.zip and b/app-dev/devops-and-containers/oke/oke-rm/oke/oke.zip differ diff --git a/app-dev/devops-and-containers/oke/oke-rm/oke/policies.tf b/app-dev/devops-and-containers/oke/oke-rm/oke/policies.tf index 5c482ba56..cd06c8b02 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/oke/policies.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/oke/policies.tf @@ -36,25 +36,25 @@ resource "oci_identity_policy" "cluster_autoscaler_policy_all" { compartment_id = var.oke_compartment_id description = "Policies for the OKE cluster autoscaler using workload identity" name = "${var.cluster_name}-cluster-autoscaler-policies" - statements = local.cluster_autoscaler_addon_all_statements - provider = oci.home - count = local.node_pool_network_compartment_same && local.create_autoscaler_policies ? 1 : 0 + statements = local.cluster_autoscaler_addon_all_statements + provider = oci.home + count = local.node_pool_network_compartment_same && local.create_autoscaler_policies ? 1 : 0 } resource "oci_identity_policy" "cluster_autoscaler_policy_network" { compartment_id = var.network_compartment_id description = "Policies for the OKE cluster autoscaler using workload identity" name = "${var.cluster_name}-cluster-autoscaler-policies-network" - statements = local.cluster_autoscaler_addon_network_statements - provider = oci.home - count = !local.node_pool_network_compartment_same && local.create_autoscaler_policies ? 1 : 0 + statements = local.cluster_autoscaler_addon_network_statements + provider = oci.home + count = !local.node_pool_network_compartment_same && local.create_autoscaler_policies ? 1 : 0 } resource "oci_identity_policy" "cluster_autoscaler_policy_nodepool" { compartment_id = var.oke_compartment_id description = "Policies for the OKE cluster autoscaler using workload identity" name = "${var.cluster_name}-cluster-autoscaler-policies-nodepool" - statements = local.cluster_autoscaler_addon_nodepool_statements - provider = oci.home - count = !local.node_pool_network_compartment_same && local.create_autoscaler_policies ? 1 : 0 + statements = local.cluster_autoscaler_addon_nodepool_statements + provider = oci.home + count = !local.node_pool_network_compartment_same && local.create_autoscaler_policies ? 1 : 0 } \ No newline at end of file diff --git a/app-dev/devops-and-containers/oke/oke-rm/oke/provider.tf b/app-dev/devops-and-containers/oke/oke-rm/oke/provider.tf index 389c8440f..ec668120e 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/oke/provider.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/oke/provider.tf @@ -2,8 +2,8 @@ terraform { required_version = ">=1.5.0" required_providers { oci = { - source = "oracle/oci" - version = "7.22.0" + source = "oracle/oci" + version = "7.22.0" configuration_aliases = [oci.home] } } diff --git a/app-dev/devops-and-containers/oke/oke-rm/oke/variable.tf b/app-dev/devops-and-containers/oke/oke-rm/oke/variable.tf index eefa1ebb4..cdbf082a1 100644 --- a/app-dev/devops-and-containers/oke/oke-rm/oke/variable.tf +++ b/app-dev/devops-and-containers/oke/oke-rm/oke/variable.tf @@ -21,7 +21,7 @@ variable "pod_subnet_id" { default = null } variable "cp_allowed_cidr_list" { - type = list(string) + type = list(string) default = ["0.0.0.0/0"] } @@ -49,22 +49,22 @@ variable "pods_cidr" { # ADD-ONS variable "enable_cert_manager" { - type = bool + type = bool default = true } variable "enable_metrics_server" { - type = bool + type = bool default = true } variable "enable_cluster_autoscaler" { - type = bool + type = bool default = false } variable "create_autoscaler_policies" { - type = bool + type = bool default = true } @@ -85,12 +85,12 @@ variable "cluster_kms_key_id" { # OIDC variable "enable_oidc_discovery" { - type = bool + type = bool default = false } variable "enable_oidc_authentication" { - type = bool + type = bool default = false }