Clean up bootstrap shell scripts a bit.

Author: jlamillan

README.md

@@ -165,13 +165,13 @@ $ terraform output ssh_private_key > generated/instances_id_rsa
 # Retrieve public IP for etcd nodes
 $ terraform output etcd_public_ips
 # Log in as user opc to the OEL OS
-$ ssh -i `pwd`/generated/instances_id_rsa oel@ETCD_INSTANCE_IP
+$ ssh -i `pwd`/generated/instances_id_rsa opc@ETCD_INSTANCE_IP
 # Retrieve public IP for k8s masters
 $ terraform output master_public_ips
-$ ssh -i `pwd`/generated/instances_id_rsa oel@K8SMASTER_INSTANCE_IP
+$ ssh -i `pwd`/generated/instances_id_rsa opc@K8SMASTER_INSTANCE_IP
 # Retrieve public IP for k8s workers
 $ terraform output worker_public_ips
-$ ssh -i `pwd`/generated/instances_id_rsa oel@K8SWORKER_INSTANCE_IP
+$ ssh -i `pwd`/generated/instances_id_rsa opc@K8SWORKER_INSTANCE_IP
 ```
 
 ### Mandatory Input Variables:
@@ -232,7 +232,7 @@ worker_nodeport_ingress             | 10.0.0.0/16 (VCN only)  | A CIDR notation
 #### Software Versions Installed on OCI Instances
 name                                | default            | description
 ------------------------------------|--------------------|------------
-docker_ver                          | 17.03              | Version of Docker to install
+docker_ver                          | 17.03.1            | Version of Docker to install
 etcd_ver                            | v3.2.2             | Version of etcd to install
 flannel_ver                         | v0.7.1             | Version of Flannel to install
 k8s_ver                             | 1.7.4              | Version of K8s to install (master and workers)
@@ -356,7 +356,7 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details.
 * Oracle Linux Enterprise (7.4)
 * etcd - (default v3.2.2)
 * flannel - (default v0.7.1)
-* docker - (default 17.03.0-ce)
+* docker - (default 17.03.1.ce)
 * apt-transport-https - (default 1.2.20)
 * kubernetes - (default v1.7.4)
   * master(s) (`kube-apiserver`, `kube-controller-manager`, `kube-scheduler`, `kubernetes-cni`, `kubectl`)

instances/etcd/cloud_init/bootstrap.template.sh

@@ -26,7 +26,7 @@ iptables -F
 ###################
 # etcd
 
-# Get IP Adress of self
+# Get IP Address of self
 IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip addr show | grep -Po 'inet \K[\d.]+')
 SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2)
 
@@ -45,11 +45,11 @@ docker run -d \
 	-listen-peer-urls http://0.0.0.0:2380 \
 	-discovery ${etcd_discovery_url}
 
-# download etcdctl client  etcd_ver
+# Download etcdctl client etcd_ver
 curl -L --retry 3 https://github.com/coreos/etcd/releases/download/${etcd_ver}/etcd-${etcd_ver}-linux-amd64.tar.gz -o /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz
 tar zxf /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz -C /tmp/ && cp /tmp/etcd-${etcd_ver}-linux-amd64/etcd* /usr/local/bin/
 
-# Generate a flannel configuration that we will store into etcd using curl.
+# Generate a flannel configuration JSON that we will store into etcd using curl.
 cat >/tmp/flannel-network.json <<EOF
 {
   "Network": "${flannel_network_cidr}",

instances/k8smaster/scripts/setup.template.sh

@@ -11,14 +11,16 @@ export IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip add
 
 SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2)
 
-## download etcdctl client
+## etcd
 ######################################
+
+# Download etcdctl client
 curl -L --retry 3 https://github.com/coreos/etcd/releases/download/${etcd_ver}/etcd-${etcd_ver}-linux-amd64.tar.gz -o /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz
 tar zxf /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz -C /tmp/ && cp /tmp/etcd-${etcd_ver}-linux-amd64/etcd* /usr/local/bin/
 
-# wait for etcd to become active (through the LB)
+# Wait for etcd to become active (through the LB)
 until [ $(/usr/local/bin/etcdctl --endpoints ${etcd_lb} cluster-health | grep '^cluster ' | grep -c 'is healthy$') == "1" ]; do
-	echo "Waiting for cluster to be healthy"
+	echo "Waiting for etcd cluster to be healthy"
 	sleep 1
 done
 
@@ -31,47 +33,20 @@ echo "IP_LOCAL: $IP_LOCAL ETCD_SERVER: $ETCD_SERVER"
 envsubst </root/services/flannel.service >/etc/systemd/system/flannel.service
 systemctl daemon-reload && systemctl enable flannel && systemctl start flannel
 
-## INSTALL CNI PLUGIN
-######################################
-mkdir -p /opt/cni/bin /etc/cni/net.d
-curl -L --retry 3 https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-v0.5.2.tgz -o /tmp/cni-plugin.tar.gz
-tar zxf /tmp/cni-plugin.tar.gz -C /opt/cni/bin/
-printf '{\n    "name": "podnet",\n    "type": "flannel",\n    "delegate": {\n        "isDefaultGateway": true\n    }\n}\n' >/etc/cni/net.d/10-flannel.conf
-
+# Create cni bridge interface w/ IP from flannel
 cp /root/services/cni-bridge.service /etc/systemd/system/cni-bridge.service
 cp /root/services/cni-bridge.sh /usr/local/bin/cni-bridge.sh && chmod +x /usr/local/bin/cni-bridge.sh
 systemctl enable cni-bridge && systemctl start cni-bridge
 
-# Install Docker prereqs
-until yum -y install aufs-tools cgroupfs-mount libltdl7 unzip; do sleep 1 && echo -n "."; done
-
-# Stage master certs
-unzip /tmp/k8s-certs.zip -d /etc/kubernetes/ssl/
-
-# enable ol7 addons
-yum-config-manager --disable ol7_UEKR3
-yum-config-manager --enable ol7_addons ol7_latest ol7_UEKR4 ol7_optional ol7_optional_latest
-
-# Install Docker
+## Docker
+######################################
 until yum -y install docker-engine-${docker_ver}; do sleep 1 && echo -n "."; done
 
-systemctl stop docker
-
-# Disable irqbalance for performance
-service irqbalance stop
-yum -y erase irqbalance
-
+# Configure Docker to use flannel
 rm -f /lib/systemd/system/docker.service && cat /root/services/docker.service >/lib/systemd/system/docker.service
-systemctl enable docker
 systemctl daemon-reload
-systemctl restart docker
-
-# re-enable autostart
-rm -f /usr/sbin/policy-rc.d
-
-# Add default DNS
-echo "nameserver 169.254.169.254" >>/etc/resolvconf/resolv.conf.d/base
-resolvconf -u
+systemctl enable docker
+systemctl start docker
 
 # Output /etc/environment_params
 echo "IPV4_PRIVATE_0=$IP_LOCAL" >>/etc/environment_params
@@ -97,10 +72,22 @@ setenforce 0
 systemctl stop firewalld.service
 systemctl disable firewalld.service
 
+# Configure pod network:
+mkdir -p /etc/cni/net.d
+cat >/etc/cni/net.d/10-flannel.conf <<EOF
+{
+	"name": "podnet",
+	"type": "flannel",
+	"delegate": {
+		"isDefaultGateway": true
+	}
+}
+EOF
+
 ## Install kubelet, kubectl, and kubernetes-cni
 ###############################################
 yum-config-manager --add-repo http://yum.kubernetes.io/repos/kubernetes-el7-x86_64
-until yum install -y kubelet-${k8s_ver}-0 kubectl-${k8s_ver}-0; do sleep 1 && echo -n ".";done
+until yum install -y kubelet-${k8s_ver}-0 kubectl-${k8s_ver}-0 kubernetes-cni; do sleep 1 && echo -n ".";done
 
 # Pull etcd docker image from registry
 docker pull quay.io/coreos/etcd:${etcd_ver}
@@ -116,9 +103,7 @@ docker run -d \
 	-discovery ${etcd_discovery_url} \
 	--proxy on
 
-## Kubelet for the master
-systemctl stop kubelet
-rm /lib/systemd/system/kubelet.service
+## kubelet for the master
 systemctl daemon-reload
 sed -e "s/__FQDN_HOSTNAME__/$FQDN_HOSTNAME/g" /root/services/kubelet.service >/etc/systemd/system/kubelet.service
 systemctl daemon-reload
@@ -127,7 +112,7 @@ systemctl start kubelet
 
 until kubectl get all; do sleep 1 && echo -n "."; done
 
-## wait for k8smaster to be healthy. possible race on pod networks otherwise
+## Wait for k8s master to be available. There is a possible race on pod networks otherwise.
 until [ "$(curl localhost:8080/healthz 2>/dev/null)" == "ok" ]; do
 	sleep 3
 done
@@ -138,4 +123,4 @@ kubectl create -f /root/services/kube-dns.yaml
 ## install kubernetes-dashboard
 kubectl create -f /root/services/kubernetes-dashboard.yaml
 
-echo "Finished running setup.sh"
+echo "Finished running setup.sh"

instances/k8sworker/main.tf

@@ -23,7 +23,7 @@ resource "oci_core_instance" "TFInstanceK8sWorker" {
     when = "destroy"
 
     inline = [
-      "nodeName=`getent hosts $(ip route get 1 | awk '{print $NF;exit}') | awk '{print $2}'`",
+      "nodeName=`getent hosts $(/usr/sbin/ip route get 1 | awk '{print $NF;exit}') | awk '{print $2}'`",
       "[ -e /usr/bin/kubectl ] && sudo kubectl --kubeconfig /etc/kubernetes/manifests/worker-kubeconfig.yaml drain $nodeName --force",
       "[ -e /usr/bin/kubectl ] && sudo kubectl --kubeconfig /etc/kubernetes/manifests/worker-kubeconfig.yaml delete node $nodeName",
       "exit 0",

instances/k8sworker/scripts/setup.template.sh

@@ -4,10 +4,10 @@ EXTERNAL_IP=$(curl -s -m 10 http://whatismyip.akamai.com/)
 NAMESPACE=$(echo -n "${domain_name}" | sed "s/\.oraclevcn\.com//g")
 FQDN_HOSTNAME=$(getent hosts $(ip route get 1 | awk '{print $NF;exit}') | awk '{print $2}')
 
-# pull instance metadata
+# Pull instance metadata
 curl -sL --retry 3 http://169.254.169.254/opc/v1/instance/ | tee /tmp/instance_meta.json
 
-## create policy file that blocks autostart of services on install
+## Create policy file that blocks autostart of services on install
 printf '#!/bin/sh\necho "All runlevel operations denied by policy" >&2\nexit 101\n' >/tmp/policy-rc.d && chmod +x /tmp/policy-rc.d
 export K8S_API_SERVER_LB=${master_lb}
 export ETCD_LB=${etcd_lb}
@@ -19,14 +19,16 @@ export IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip add
 SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2)
 export WORKER_IP=$IP_LOCAL
 
-## download etcdctl client
+## etcd
 ######################################
+
+# Download etcdctl client
 curl -L --retry 3 https://github.com/coreos/etcd/releases/download/${etcd_ver}/etcd-${etcd_ver}-linux-amd64.tar.gz -o /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz
 tar zxf /tmp/etcd-${etcd_ver}-linux-amd64.tar.gz -C /tmp/ && cp /tmp/etcd-${etcd_ver}-linux-amd64/etcd* /usr/local/bin/
 
-# wait for etcd to become active (through the LB)
+# Wait for etcd to become active (through the LB)
 until [ $(/usr/local/bin/etcdctl --endpoints ${etcd_lb} cluster-health | grep '^cluster ' | grep -c 'is healthy$') == "1" ]; do
-	echo "Waiting for cluster to be healthy"
+	echo "Waiting for etcd cluster to be healthy"
 	sleep 1
 done
 
@@ -39,47 +41,21 @@ echo "IP_LOCAL: $IP_LOCAL ETCD_SERVER: $ETCD_SERVER"
 envsubst </root/services/flannel.service >/etc/systemd/system/flannel.service
 systemctl daemon-reload && systemctl enable flannel && systemctl start flannel
 
-## INSTALL CNI PLUGIN
+## Create cni bridge interface w/ IP from flannel
 ######################################
-mkdir -p /opt/cni/bin /etc/cni/net.d
-curl -L --retry 3 https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-v0.5.2.tgz -o /tmp/cni-plugin.tar.gz
-tar zxf /tmp/cni-plugin.tar.gz -C /opt/cni/bin/
-printf '{\n    "name": "podnet",\n    "type": "flannel",\n    "delegate": {\n        "isDefaultGateway": true\n    }\n}\n' >/etc/cni/net.d/10-flannel.conf
 cp /root/services/cni-bridge.service /etc/systemd/system/cni-bridge.service
 cp /root/services/cni-bridge.sh /usr/local/bin/cni-bridge.sh && chmod +x /usr/local/bin/cni-bridge.sh
 systemctl enable cni-bridge && systemctl start cni-bridge
 
-###################################### DOCKER ######################################
-
-## Install Docker prereqs
+## Docker
 ######################################
-until yum -y install aufs-tools cgroupfs-mount libltdl7 unzip; do sleep && echo -n "."; done
-
-# Stage worker certs
-unzip /tmp/k8s-certs.zip -d /etc/kubernetes/ssl/
-
-# enable ol7 addons
-yum-config-manager --disable ol7_UEKR3
-yum-config-manager --enable ol7_addons ol7_latest ol7_UEKR4 ol7_optional ol7_optional_latest
-
-# Install Docker
 until yum -y install docker-engine-${docker_ver}; do sleep 1 && echo -n "."; done
-systemctl stop docker
-
-# Disable irqbalance for performance
-service irqbalance stop
-yum -y erase irqbalance
 
+# Configure Docker to use flannel
 rm -f /lib/systemd/system/docker.service && cat /root/services/docker.service >/lib/systemd/system/docker.service
-systemctl enable docker
 systemctl daemon-reload
-systemctl restart docker
-
-
-## Add default DNS
-######################################
-echo "nameserver 169.254.169.254" >>/etc/resolvconf/resolv.conf.d/base
-resolvconf -u
+systemctl enable docker
+systemctl start docker
 
 ## Output /etc/environment_params
 ######################################
@@ -92,7 +68,6 @@ echo "FQDN_HOSTNAME=$FQDN_HOSTNAME" >>/etc/environment_params
 ######################################
 iptables -F
 
-
 cat <<EOF > /etc/yum.repos.d/kubernetes.repo
 [kubernetes]
 name=Kubernetes
@@ -109,17 +84,22 @@ setenforce 0
 systemctl stop firewalld.service
 systemctl disable firewalld.service
 
+# Configure pod network:
+mkdir -p /etc/cni/net.d
+cat >/etc/cni/net.d/10-flannel.conf <<EOF
+{
+	"name": "podnet",
+	"type": "flannel",
+	"delegate": {
+		"isDefaultGateway": true
+	}
+}
+EOF
+
 ## Install kubelet, kubectl, and kubernetes-cni
 ###############################################
 yum-config-manager --add-repo http://yum.kubernetes.io/repos/kubernetes-el7-x86_64
-until yum install -y kubelet-${k8s_ver}-0 kubectl-${k8s_ver}-0; do sleep 1 && echo -n ".";done
-
-until systemctl stop kubelet; do sleep 1; done
-mkdir -p /opt/cni/bin /etc/cni/net.d
-tar zxf /tmp/cni-plugin.tar.gz -C /opt/cni/bin/
-printf '{\n    "name": "podnet",\n    "type": "flannel",\n    "delegate": {\n        "isDefaultGateway": true\n    }\n}\n' >/etc/cni/net.d/10-flannel.conf
-
-###################################### ETCD ######################################
+until yum install -y kubelet-${k8s_ver}-0 kubectl-${k8s_ver}-0 kubernetes-cni; do sleep 1 && echo -n ".";done
 
 ## Pull etcd docker image from registry
 docker pull quay.io/coreos/etcd:${etcd_ver}
@@ -140,9 +120,8 @@ docker run -d \
 sed -e "s/__FQDN_HOSTNAME__/$FQDN_HOSTNAME/g" /etc/kubernetes/manifests/kube-proxy.yaml >/tmp/kube-proxy.yaml
 cat /tmp/kube-proxy.yaml >/etc/kubernetes/manifests/kube-proxy.yaml
 
-## Kubelet for the worker
+## kubelet for the worker
 ######################################
-rm /lib/systemd/system/kubelet.service
 systemctl daemon-reload
 
 AVAILABILITY_DOMAIN=$(jq -r '.availabilityDomain' /tmp/instance_meta.json | sed 's/:/-/g')
@@ -160,7 +139,7 @@ sed -e "s/__FQDN_HOSTNAME__/$FQDN_HOSTNAME/g" \
     -e "s/__NODE_SHAPE__/$NODE_SHAPE/g" \
     /root/services/kubelet.service > /etc/systemd/system/kubelet.service
 
-## wait for k8smaster to be available. possible race on pod networks otherwise
+## Wait for k8s master to be available. There is a possible race on pod networks otherwise.
 until [ "$(curl -k --cert /etc/kubernetes/ssl/apiserver.pem --key /etc/kubernetes/ssl/apiserver-key.pem $K8S_API_SERVER_LB/healthz 2>/dev/null)" == "ok" ]; do
 	sleep 3
 done