Merge pull request #350 from pbx0/script-updates

support self-hosted etcd in quickstart scripts

Author: aaronlevy

hack/quickstart/init-master.sh

@@ -6,6 +6,7 @@ REMOTE_PORT=${REMOTE_PORT:-22}
 CLUSTER_DIR=${CLUSTER_DIR:-cluster}
 IDENT=${IDENT:-${HOME}/.ssh/id_rsa}
 SSH_OPTS=${SSH_OPTS:-}
+SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
 
 BOOTKUBE_REPO=${BOOTKUBE_REPO:-quay.io/coreos/bootkube}
 BOOTKUBE_VERSION=${BOOTKUBE_VERSION:-v0.3.9}
@@ -37,16 +38,25 @@ function init_master_node() {
     systemctl daemon-reload
     systemctl stop update-engine; systemctl mask update-engine
 
+    etcd_render_flags=""
+    etcd_start_flags=""
+
     # Start etcd.
-    configure_etcd
-    systemctl enable etcd-member; sudo systemctl start etcd-member
+    if [ "$SELF_HOST_ETCD" = true ] ; then
+        echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
+        etcd_render_flags="--etcd-servers=http://10.3.0.15:2379 --experimental-self-hosted-etcd"
+        etcd_start_flags="--etcd-server=http://${COREOS_PRIVATE_IPV4}:12379 --experimental-self-hosted-etcd"
+    else
+        configure_etcd
+        systemctl enable etcd-member; sudo systemctl start etcd-member
+    fi
 
     # Render cluster assets
     /usr/bin/rkt run \
         --volume home,kind=host,source=/home/core \
         --mount volume=home,target=/core \
         --trust-keys-from-https --net=host ${BOOTKUBE_REPO}:${BOOTKUBE_VERSION} --exec \
-        /bootkube -- render --asset-dir=/core/assets --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443
+        /bootkube -- render --asset-dir=/core/assets --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443 ${etcd_render_flags}
 
     # Move the local kubeconfig into expected location
     chown -R core:core /home/core/assets
@@ -61,8 +71,10 @@ function init_master_node() {
     /usr/bin/rkt run \
         --volume home,kind=host,source=/home/core \
         --mount volume=home,target=/core \
+        --volume manifests,kind=host,source=/etc/kubernetes/manifests \
+        --mount volume=manifests,target=/etc/kubernetes/manifests \
         --net=host ${BOOTKUBE_REPO}:${BOOTKUBE_VERSION} --exec \
-        /bootkube -- start --asset-dir=/core/assets
+        /bootkube -- start --asset-dir=/core/assets ${etcd_start_flags}
 }
 
 [ "$#" == 1 ] || usage
@@ -81,7 +93,7 @@ if [ "${REMOTE_HOST}" != "local" ]; then
 
     # Copy self to remote host so script can be executed in "local" mode
     scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} core@${REMOTE_HOST}:/home/core/init-master.sh
-    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo BOOTKUBE_REPO=${BOOTKUBE_REPO} BOOTKUBE_VERSION=${BOOTKUBE_VERSION} /home/core/init-master.sh local"
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo BOOTKUBE_REPO=${BOOTKUBE_REPO} BOOTKUBE_VERSION=${BOOTKUBE_VERSION} SELF_HOST_ETCD=${SELF_HOST_ETCD} /home/core/init-master.sh local"
 
     # Copy assets from remote host to a local directory. These can be used to launch additional nodes & contain TLS assets
     mkdir ${CLUSTER_DIR}

hack/quickstart/init-worker.sh

@@ -13,33 +13,17 @@ function usage() {
     exit 1
 }
 
-function extract_master_endpoint (){
-    grep 'certificate-authority-data' ${KUBECONFIG} | awk '{print $2}' | base64 -d > /home/core/ca.crt
-    grep 'client-certificate-data' ${KUBECONFIG} | awk '{print $2}'| base64 -d > /home/core/client.crt
-    grep 'client-key-data' ${KUBECONFIG} | awk '{print $2}'| base64 -d > /home/core/client.key
-
-    MASTER_PUB="$(awk '/server:/ {print $2}' ${KUBECONFIG} | awk -F/ '{print $3}' | awk -F: '{print $1}')"
-    # TODO (aaron): The -k was added with the gce conformance tests - figure out why it's needed here.
-    #     The certs are seemingly signed correctly, but says no SAN for MASTER_PUB
-    MASTER_PRIV=$(curl -k https://${MASTER_PUB}:443/api/v1/namespaces/default/endpoints/kubernetes \
-        --cacert /home/core/ca.crt --cert /home/core/client.crt --key /home/core/client.key \
-        | jq -r '.subsets[0].addresses[0].ip')
-    rm -f /home/core/client.crt /home/core/client.key
-}
-
 # Initialize a worker node
 function init_worker_node() {
-    extract_master_endpoint
 
     # Setup kubeconfig
     mkdir -p /etc/kubernetes
     cp ${KUBECONFIG} /etc/kubernetes/kubeconfig
-    # Pulled out of the kubeconfig in extract_master_endpoint. Other installations should
-    # place the root CA here manually.
-    mv /home/core/ca.crt /etc/kubernetes/ca.crt
+    # Pulled out of the kubeconfig. Other installations should place the root
+    # CA here manually.
+    grep 'certificate-authority-data' ${KUBECONFIG} | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt
 
-    sed "s/{{apiserver}}/${MASTER_PRIV}/" /home/core/kubelet.worker > /etc/systemd/system/kubelet.service
-    rm /home/core/kubelet.worker
+    mv /home/core/kubelet.worker /etc/systemd/system/kubelet.service
 
     # Start services
     systemctl daemon-reload

hack/quickstart/kubelet.master

@@ -13,8 +13,8 @@ ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
 ExecStartPre=/bin/mkdir -p /var/lib/cni
 ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
 ExecStart=/usr/lib/coreos/kubelet-wrapper \
-  --api-servers=https://${COREOS_PRIVATE_IPV4}:443 \
   --kubeconfig=/etc/kubernetes/kubeconfig \
+  --require-kubeconfig \
   --client-ca-file=/etc/kubernetes/ca.crt \
   --anonymous-auth=false \
   --cni-conf-dir=/etc/kubernetes/cni/net.d \

hack/quickstart/kubelet.worker

@@ -11,8 +11,8 @@ ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
 ExecStartPre=/bin/mkdir -p /var/lib/cni
 ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
 ExecStart=/usr/lib/coreos/kubelet-wrapper \
-  --api-servers=https://{{apiserver}}:443 \
   --kubeconfig=/etc/kubernetes/kubeconfig \
+  --require-kubeconfig \
   --client-ca-file=/etc/kubernetes/ca.crt \
   --anonymous-auth=false \
   --cni-conf-dir=/etc/kubernetes/cni/net.d \

hack/tests/conformance-gce.sh

@@ -31,7 +31,11 @@ BOOTKUBE_REPO=${BOOTKUBE_REPO:-}
 BOOTKUBE_VERSION=${BOOTKUBE_VERSION:-}
 COREOS_CHANNEL=${COREOS_CHANNEL:-'coreos-stable'}
 WORKER_COUNT=4
+SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
+
 GCE_PREFIX=${GCE_PREFIX:-'bootkube-ci'}
+GCE_SERVICE_ACCOUNT=${GCE_SERVICE_ACCOUNT:-'bootkube-ci'}
+GCE_PROJECT=${GCE_PROJECT:-coreos-gce-testing}
 
 function cleanup {
     gcloud compute instances delete --quiet --zone us-central1-a ${GCE_PREFIX}-m1 || true
@@ -45,8 +49,8 @@ function cleanup {
 function init {
     curl https://sdk.cloud.google.com | bash
     source ~/.bashrc
-    gcloud config set project coreos-gce-testing
-    gcloud auth activate-service-account ${GCE_PREFIX}@coreos-gce-testing.iam.gserviceaccount.com --key-file=/build/keyfile
+    gcloud config set project ${GCE_PROJECT}
+    gcloud auth activate-service-account ${GCE_SERVICE_ACCOUNT}@${GCE_PROJECT}.iam.gserviceaccount.com --key-file=/build/keyfile
     apt-get update && apt-get install -y jq
 
     ssh-keygen -t rsa -f /root/.ssh/id_rsa -N ""
@@ -64,7 +68,7 @@ function add_master {
 
     MASTER_IP=$(gcloud compute instances list ${GCE_PREFIX}-m1 --format=json | jq --raw-output '.[].networkInterfaces[].accessConfigs[].natIP')
     cd /build/bootkube/hack/quickstart && SSH_OPTS="-o StrictHostKeyChecking=no" \
-        CLUSTER_DIR=/build/cluster BOOTKUBE_REPO=${BOOTKUBE_REPO} BOOTKUBE_VERSION=${BOOTKUBE_VERSION} ./init-master.sh ${MASTER_IP}
+        CLUSTER_DIR=/build/cluster BOOTKUBE_REPO=${BOOTKUBE_REPO} BOOTKUBE_VERSION=${BOOTKUBE_VERSION} SELF_HOST_ETCD=${SELF_HOST_ETCD} ./init-master.sh ${MASTER_IP}
 }
 
 function add_workers {
@@ -111,6 +115,8 @@ else
         "--mount volume=keyfile,target=/build/keyfile " \
     )
 
+    #TODO(pb): See if there is a way to make the --inherit-env option replace
+    #passing all the variables manually. 
     sudo rkt run --insecure-options=image ${RKT_OPTS} docker://golang:1.7.4 --exec /bin/bash -- -c \
-        "IN_CONTAINER=true BOOTKUBE_REPO=${BOOTKUBE_REPO} BOOTKUBE_VERSION=${BOOTKUBE_VERSION} COREOS_CHANNEL=${COREOS_CHANNEL} /build/bootkube/hack/tests/$(basename $0)"
+        "IN_CONTAINER=true BOOTKUBE_REPO=${BOOTKUBE_REPO} BOOTKUBE_VERSION=${BOOTKUBE_VERSION} COREOS_CHANNEL=${COREOS_CHANNEL} GCE_PREFIX=${GCE_PREFIX} GCE_SERVICE_ACCOUNT=${GCE_SERVICE_ACCOUNT} GCE_PROJECT=${GCE_PROJECT} SELF_HOST_ETCD=${SELF_HOST_ETCD} /build/bootkube/hack/tests/$(basename $0)"
 fi