migrate to kube-flannel

worker nodes no longer need to communicate directly with etcd

Author: Timothy J Fontaine

instances/etcd/cloud_init/bootstrap.template.sh

@@ -74,27 +74,12 @@ docker run -d \
 	-listen-peer-urls http://0.0.0.0:2380 \
 	-discovery ${etcd_discovery_url}
 
-# Generate a flannel configuration JSON that we will store into etcd using curl.
-cat >/tmp/flannel-network.json <<EOF
-{
-  "Network": "${flannel_network_cidr}",
-  "Subnetlen": ${flannel_network_subnetlen},
-  "Backend": {
-    "Type": "${flannel_backend}",
-    "VNI" : 1
-  }
-}
-EOF
-
 # wait for etcd to become active
 while ! curl -sf -o /dev/null http://$FQDN_HOSTNAME:2379/v2/keys/; do
 	sleep 1
 	echo "Try again"
 done
 
-# put the flannel config in etcd
-curl -sf -L http://$FQDN_HOSTNAME:2379/v2/keys/flannel/network/config -X PUT --data-urlencode value@/tmp/flannel-network.json
-
 # Download etcdctl client etcd_ver
 while ! curl -L https://github.com/coreos/etcd/releases/download/${etcd_ver}/etcd-${etcd_ver}-linux-amd64.tar.gz -o /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz; do
 	sleep 1

instances/etcd/datasources.tf

@@ -17,9 +17,6 @@ data "template_file" "etcd-bootstrap" {
     domain_name               = "${var.domain_name}"
     docker_ver                = "${var.docker_ver}"
     etcd_ver                  = "${var.etcd_ver}"
-    flannel_network_cidr      = "${var.flannel_network_cidr}"
-    flannel_network_subnetlen = "${var.flannel_network_subnetlen}"
-    flannel_backend           = "${var.flannel_backend}"
     docker_max_log_size       = "${var.etcd_docker_max_log_size}"
     docker_max_log_files      = "${var.etcd_docker_max_log_files}"
     etcd_discovery_url        = "${file("${path.root}/generated/discovery${var.etcd_discovery_url}")}"

instances/etcd/variables.tf

@@ -32,9 +32,6 @@ variable "etcd_ver" {
 }
 
 variable "tenancy_ocid" {}
-variable "flannel_network_cidr" {}
-variable "flannel_network_subnetlen" {}
-variable "flannel_backend" {}
 variable "etcd_discovery_url" {}
 
 variable "count" {

instances/k8smaster/cloud_init/bootstrap.template.yaml

@@ -67,21 +67,6 @@ write_files:
     encoding: "gzip+base64"
     content: |
       ${volume_provisioner_secret_content}
-  - path: "/root/services/flannel.service"
-    permissions: "0600"
-    encoding: "gzip+base64"
-    content: |
-      ${flannel_service_content}
-  - path: "/root/services/cni-bridge.service"
-    permissions: "0600"
-    encoding: "gzip+base64"
-    content: |
-      ${cnibridge_service_content}
-  - path: "/root/services/cni-bridge.sh"
-    permissions: "0600"
-    encoding: "gzip+base64"
-    content: |
-      ${cnibridge_sh_content}
   - path: "/etc/kubernetes/manifests/master-kubeconfig.yaml"
     permissions: "0755"
     encoding: "gzip+base64"

instances/k8smaster/datasources.tf

@@ -17,6 +17,8 @@ data "template_file" "setup-template" {
     docker_ver                 = "${var.docker_ver}"
     etcd_ver                   = "${var.etcd_ver}"
     flannel_ver                = "${var.flannel_ver}"
+    flannel_network_cidr       = "${var.flannel_network_cidr}"
+    flannel_backend            = "${var.flannel_backend}"
     k8s_ver                    = "${var.k8s_ver}"
     docker_max_log_size        = "${var.master_docker_max_log_size}"
     docker_max_log_files       = "${var.master_docker_max_log_files}"
@@ -59,7 +61,8 @@ data "template_file" "kube-controller-manager" {
   template = "${file("${path.module}/manifests/kube-controller-manager.yaml")}"
 
   vars = {
-    k8s_ver = "${var.k8s_ver}"
+    k8s_ver              = "${var.k8s_ver}"
+    flannel_network_cidr = "${var.flannel_network_cidr}"
   }
 }
 
@@ -76,7 +79,8 @@ data "template_file" "kube-proxy" {
   template = "${file("${path.module}/manifests/kube-proxy.yaml")}"
 
   vars = {
-    k8s_ver = "${var.k8s_ver}"
+    k8s_ver              = "${var.k8s_ver}"
+    flannel_network_cidr = "${var.flannel_network_cidr}"
   }
 }
 
@@ -104,18 +108,6 @@ data "template_file" "master-kubeconfig" {
   template = "${file("${path.module}/manifests/master-kubeconfig.template.yaml")}"
 }
 
-data "template_file" "flannel-service" {
-  template = "${file("${path.module}/scripts/flannel.service")}"
-}
-
-data "template_file" "cnibridge-service" {
-  template = "${file("${path.module}/scripts/cni-bridge.service")}"
-}
-
-data "template_file" "cnibridge-sh" {
-  template = "${file("${path.module}/scripts/cni-bridge.sh")}"
-}
-
 data "template_file" "token_auth_file" {
   template = "${file("${path.module}/scripts/token_auth.csv")}"
 
@@ -144,9 +136,6 @@ data "template_file" "kube_master_cloud_init_file" {
     api-server-key-content                   = "${base64gzip(var.api_server_private_key_pem)}"
     api-server-cert-content                  = "${base64gzip(var.api_server_cert_pem)}"
     api-token_auth_template_content          = "${base64gzip(data.template_file.token_auth_file.rendered)}"
-    flannel_service_content                  = "${base64gzip(data.template_file.flannel-service.rendered)}"
-    cnibridge_service_content                = "${base64gzip(data.template_file.cnibridge-service.rendered)}"
-    cnibridge_sh_content                     = "${base64gzip(data.template_file.cnibridge-sh.rendered)}"
     cloud_provider_secret_content            = "${base64gzip(var.cloud_controller_secret)}"
     flexvolume_driver_secret_content         = "${base64gzip(var.flexvolume_driver_secret)}"
     volume_provisioner_secret_content        = "${base64gzip(var.volume_provisioner_secret)}"

instances/k8smaster/manifests/kube-controller-manager.yaml

@@ -12,12 +12,13 @@ spec:
     - /hyperkube
     - controller-manager
     - --master=http://127.0.0.1:8080
-    - --cluster-cidr=10.99.0.0/16
+    - --cluster-cidr=${flannel_network_cidr}
     - --cluster-name=dev-cluster
     - --leader-elect=true
     - --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem
     - --root-ca-file=/etc/kubernetes/ssl/ca.pem
     - --v=2
+    - --allocate-node-cidrs=true
     livenessProbe:
       httpGet:
         host: 127.0.0.1

instances/k8smaster/manifests/kube-proxy.yaml

@@ -18,7 +18,7 @@ spec:
     - proxy
     - --master=http://127.0.0.1:8080
     - --proxy-mode=iptables
-    - --cluster-cidr=10.99.0.0/16
+    - --cluster-cidr=${flannel_network_cidr}
     securityContext:
       privileged: true
     volumeMounts: