Make Load Balancer for Kubernetes Master(s) an optional parameter. Resolves #125. Resolves #137.

Author: xinnong-wang

docs/input-variables.md

@@ -21,6 +21,10 @@ name                                | default                 | description
 etcdShape                           | VM.Standard1.1          | OCI shape for etcd nodes
 k8sMasterShape                      | VM.Standard1.1          | OCI shape for k8s master(s)
 k8sWorkerShape                      | VM.Standard1.2          | OCI shape for k8s worker(s)
+master_oci_lb_enabled               | "true"                  | enable/disable the k8s master oci load balancer. "true": use oci load balancer for k8s master . "false": use a reverse proxy as a software load balancer for k8s masters.
+etcdLBShape                         | 100Mbps                 | etcd cluster OCI Load Balancer shape / bandwidth
+etcd_lb_enabled                     | "true"                  | enable/disable the etcd load balancer. "true" use the etcd load balancer ip, "false" use a list of etcd instance ips
+k8sMasterLBShape                    | 100Mbps                 | Kubernetes Master OCI Load Balancer shape / bandwidth
 k8sMasterAd1Count                   | 1                       | number of k8s masters to create in Availability Domain 1
 k8sMasterAd2Count                   | 0                       | number of k8s masters to create in Availability Domain 2
 k8sMasterAd3Count                   | 0                       | number of k8s masters to create in Availability Domain 3
@@ -33,11 +37,8 @@ etcdAd3Count                        | 0                       | number of etcd n
 worker_iscsi_volume_create          | "false"                 | boolean flag indicating whether or not to attach an iSCSI volume to attach to each worker node
 worker_iscsi_volume_size            | unset                   | optional size of an iSCSI volume to attach to each worker
 worker_iscsi_volume_mount           | /var/lib/docker         | optional mount path of iSCSI volume when worker_iscsi_volume_size is set
-etcd_iscsi_volume_create            | "false"                 | boolean flag indicating whether or not to attach an iSCSI volume to attach to each etcd node
+etcd_iscsi_volume_create            | false                   | boolean flag indicating whether or not to attach an iSCSI volume to attach to each etcd node
 etcd_iscsi_volume_size              | 50                      | size in GBs of volume when etcd_iscsi_volume_create is set
-etcd_lb_enabled                     | "true"                  | enable/disable the etcd load balancer. "true" use the etcd load balancer ip, "false" use a list of etcd instance ips
-etcdLBShape                         | 100Mbps                 | etcd cluster OCI Load Balancer shape / bandwidth
-k8sMasterLBShape                    | 100Mbps                 | Kubernetes Master OCI Load Balancer shape / bandwidth
 
 ### TLS Certificates & SSH key pair
 name                                | default                 | description

instances/etcd/main.tf

@@ -6,15 +6,15 @@ resource "oci_core_instance" "TFInstanceEtcd" {
   count               = "${var.count}"
   availability_domain = "${var.availability_domain}"
   compartment_id      = "${var.compartment_ocid}"
-  display_name        = "${var.label_prefix}${var.display_name}-${count.index}"
-  hostname_label      = "${var.hostname_label}-${count.index}"
+  display_name        = "${var.label_prefix}${var.display_name_prefix}-${count.index}"
+  hostname_label      = "${var.hostname_label_prefix}-${count.index}"
   image               = "${lookup(data.oci_core_images.ImageOCID.images[0], "id")}"
   shape               = "${var.shape}"
 
   create_vnic_details {
     subnet_id         = "${var.subnet_id}"
-    display_name      = "${var.label_prefix}${var.display_name}-${count.index}"
-    hostname_label    = "${var.hostname_label}-${count.index}"
+    display_name      = "${var.label_prefix}${var.display_name_prefix}-${count.index}"
+    hostname_label    = "${var.hostname_label_prefix}-${count.index}"
     assign_public_ip  = "${(var.control_plane_subnet_access == "private") ? "false" : "true"}"
     private_ip        = "${var.assign_private_ip == "true" ? cidrhost(lookup(var.network_cidrs,var.subnet_name), count.index+2) : ""}"
   },

instances/etcd/variables.tf

@@ -1,28 +1,10 @@
 variable "network_cidrs" {
   type = "map"
-  default = {
-    VCN-CIDR          = "10.0.0.0/16"
-    PublicSubnetAD1   = "10.0.10.0/24"
-    PublicSubnetAD2   = "10.0.11.0/24"
-    PublicSubnetAD3   = "10.0.12.0/24"
-    etcdSubnetAD1     = "10.0.20.0/24"
-    etcdSubnetAD2     = "10.0.21.0/24"
-    etcdSubnetAD3     = "10.0.22.0/24"
-    masterSubnetAD1   = "10.0.30.0/24"
-    masterSubnetAD2   = "10.0.31.0/24"
-    masterSubnetAD3   = "10.0.32.0/24"
-    workerSubnetAD1   = "10.0.40.0/24"
-    workerSubnetAD2   = "10.0.41.0/24"
-    workerSubnetAD3   = "10.0.42.0/24"
-    k8sCCMLBSubnetAD1 = "10.0.50.0/24"
-    k8sCCMLBSubnetAD2 = "10.0.51.0/24"
-    k8sCCMLBSubnetAD3 = "10.0.52.0/24"
-  }
 }
 variable "availability_domain" {}
 variable "compartment_ocid" {}
-variable "display_name" {}
-variable "hostname_label" {}
+variable "display_name_prefix" {}
+variable "hostname_label_prefix" {}
 
 variable "shape" {
   default = "VM.Standard1.1"

instances/k8smaster/main.tf

@@ -10,7 +10,14 @@ resource "oci_core_instance" "TFInstanceK8sMaster" {
   hostname_label      = "${var.hostname_label_prefix}-${count.index}"
   image               = "${lookup(data.oci_core_images.ImageOCID.images[0], "id")}"
   shape               = "${var.shape}"
-  subnet_id           = "${var.subnet_id}"
+
+  create_vnic_details {
+    subnet_id        = "${var.subnet_id}"
+    display_name     = "${var.label_prefix}${var.display_name_prefix}-${count.index}"
+    hostname_label   = "${var.hostname_label_prefix}-${count.index}"
+    assign_public_ip = "${(var.control_plane_subnet_access == "private") ? "false" : "true"}"
+    private_ip       = "${var.assign_private_ip == "true" ? cidrhost(lookup(var.network_cidrs,var.subnet_name), count.index+2) : ""}"
+  }
 
   extended_metadata {
     roles               = "masters"

instances/k8smaster/variables.tf

@@ -9,7 +9,16 @@ variable "count" {
   default = "1"
 }
 
+variable "control_plane_subnet_access" {
+  description = "Whether instances in the control plane are launched in a public or private subnets"
+  default     = "public"
+}
+
+variable "network_cidrs" {
+  type = "map"
+}
 variable "subnet_id" {}
+variable "subnet_name" {}
 variable "domain_name" {}
 variable "shape" {}
 variable "tenancy_ocid" {}
@@ -81,6 +90,7 @@ variable "flexvolume_driver_secret" {}
 variable "volume_provisioner_version" {}
 variable "volume_provisioner_secret" {}
 
-
-
-
+variable "assign_private_ip" {
+  description = "Assign a static private ip based on CIDR block for that AD"
+  default = false
+}

instances/k8sworker/cloud_init/bootstrap.template.yaml

@@ -75,6 +75,7 @@ write_files:
     content: |
       ${api-server-key-content}
 
+${reverse_proxy-content}
 
 runcmd:
  - echo "Running k8s init..."

instances/k8sworker/datasources.tf

@@ -13,18 +13,19 @@ data "template_file" "setup-template" {
   template = "${file("${path.module}/scripts/setup.template.sh")}"
 
   vars = {
-    master_lb                  = "${var.master_lb}"
-    domain_name                = "${var.domain_name}"
-    docker_ver                 = "${var.docker_ver}"
-    etcd_ver                   = "${var.etcd_ver}"
-    flannel_ver                = "${var.flannel_ver}"
-    k8s_ver                    = "${var.k8s_ver}"
-    docker_max_log_size        = "${var.worker_docker_max_log_size}"
-    docker_max_log_files       = "${var.worker_docker_max_log_files}"
-    etcd_discovery_url         = "${file("${path.root}/generated/discovery${var.etcd_discovery_url}")}"
-    etcd_endpoints             = "${var.etcd_endpoints}"
-    worker_iscsi_volume_mount  = "${var.worker_iscsi_volume_mount}"
-    flexvolume_driver_version  = "${var.flexvolume_driver_version}"
+    master_lb                 = "${var.master_lb}"
+    domain_name               = "${var.domain_name}"
+    docker_ver                = "${var.docker_ver}"
+    etcd_ver                  = "${var.etcd_ver}"
+    flannel_ver               = "${var.flannel_ver}"
+    k8s_ver                   = "${var.k8s_ver}"
+    docker_max_log_size       = "${var.worker_docker_max_log_size}"
+    docker_max_log_files      = "${var.worker_docker_max_log_files}"
+    etcd_discovery_url        = "${file("${path.root}/generated/discovery${var.etcd_discovery_url}")}"
+    etcd_endpoints            = "${var.etcd_endpoints}"
+    worker_iscsi_volume_mount = "${var.worker_iscsi_volume_mount}"
+    flexvolume_driver_version = "${var.flexvolume_driver_version}"
+    reverse_proxy_setup       = "${var.reverse_proxy_setup}"
   }
 }
 
@@ -97,6 +98,7 @@ data "template_file" "kube_worker_cloud_init_file" {
     ca-key-content                     = "${base64gzip(var.root_ca_key)}"
     api-server-key-content             = "${base64gzip(var.api_server_private_key_pem)}"
     api-server-cert-content            = "${base64gzip(var.api_server_cert_pem)}"
+    reverse_proxy-content              = "${var.reverse_proxy_clount_init}"
   }
 }
 

instances/k8sworker/scripts/setup.template.sh

@@ -234,6 +234,7 @@ sed -e "s/__FQDN_HOSTNAME__/$FQDN_HOSTNAME/g" \
     -e "s/__SWAP_OPTION__/$SWAP_OPTION/g" \
     /root/services/kubelet.service > /etc/systemd/system/kubelet.service
 
+${reverse_proxy_setup}
 ## Wait for k8s master to be available. There is a possible race on pod networks otherwise.
 until [ "$(curl -k --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem $K8S_API_SERVER_LB/healthz 2>/dev/null)" == "ok" ]; do
 	sleep 3

instances/k8sworker/variables.tf

@@ -83,3 +83,7 @@ variable "worker_iscsi_volume_mount" {
 }
 
 variable "flexvolume_driver_version" {}
+
+variable "reverse_proxy_setup" {}
+
+variable "reverse_proxy_clount_init" {}