Merge pull request #321 from ericchiang/revert-tls-bootstrapping

*: revert TLS bootstrapping

Author: aaronlevy

cmd/bootkube/render.go

@@ -27,16 +27,15 @@ var (
 	}
 
 	renderOpts struct {
-		assetDir           string
-		caCertificatePath  string
-		caPrivateKeyPath   string
-		bootstrapAuthToken string
-		etcdServers        string
-		apiServers         string
-		altNames           string
-		selfHostKubelet    bool
-		cloudProvider      string
-		selfHostedEtcd     bool
+		assetDir          string
+		caCertificatePath string
+		caPrivateKeyPath  string
+		etcdServers       string
+		apiServers        string
+		altNames          string
+		selfHostKubelet   bool
+		cloudProvider     string
+		selfHostedEtcd    bool
 	}
 )
 

hack/csrctl.sh

@@ -22,8 +22,8 @@ fi
 # Render assets
 if [ ! -d "cluster" ]; then
   ../../_output/bin/${local_os}/bootkube render --asset-dir=cluster --api-servers=https://172.17.4.101:443 ${etcd_render_flags}
-  # Add rendered bootstrap-kubeconfig to the node user-data
-  cat user-data.sample > cluster/user-data && sed 's/^/      /' cluster/auth/bootstrap-kubeconfig >> cluster/user-data
+  # Add rendered kubeconfig to the node user-data
+  cat user-data.sample > cluster/user-data && sed 's/^/      /' cluster/auth/kubeconfig >> cluster/user-data
   cp cluster/user-data{,-worker}
   cp cluster/user-data{,-controller}
   sed -i.bak -e '/--node-labels=master=true/d' cluster/user-data-worker
@@ -42,5 +42,5 @@ ssh -q -F ssh_config core@c1 "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --
 
 echo
 echo "Bootstrap complete. Access your kubernetes cluster using:"
-echo "kubectl --kubeconfig=cluster/auth/admin-kubeconfig get nodes"
+echo "kubectl --kubeconfig=cluster/auth/kubeconfig get nodes"
 echo

hack/multi-node/bootkube-up

@@ -18,8 +18,6 @@ coreos:
         ExecStartPre=/bin/mkdir -p /var/lib/cni
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
           --kubeconfig=/etc/kubernetes/kubeconfig \
-          --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig \
-          --cert-dir=/etc/kubernetes/secrets \
           --require-kubeconfig \
           --cni-conf-dir=/etc/kubernetes/cni/net.d \
           --network-plugin=cni \
@@ -38,7 +36,7 @@ coreos:
         WantedBy=multi-user.target
 
 write_files:
-  - path: "/etc/kubernetes/bootstrap-kubeconfig"
+  - path: "/etc/kubernetes/kubeconfig"
     permissions: "0644"
     owner: core
     content: |

hack/multi-node/user-data.sample

@@ -46,14 +46,12 @@ function init_master_node() {
         --volume home,kind=host,source=/home/core \
         --mount volume=home,target=/core \
         --trust-keys-from-https --net=host ${BOOTKUBE_REPO}:${BOOTKUBE_VERSION} --exec \
-        /bootkube -- render --asset-dir=/core/assets --api-servers=https://${COREOS_PRIVATE_IPV4}:443,https://${COREOS_PUBLIC_IPV4}:443
+        /bootkube -- render --asset-dir=/core/assets --api-servers=https://${COREOS_PUBLIC_IPV4}:443,https://${COREOS_PRIVATE_IPV4}:443
 
-    # Move the local bootstrap-kubeconfig into expected location
+    # Move the local kubeconfig into expected location
     chown -R core:core /home/core/assets
     mkdir -p /etc/kubernetes
-    cp /home/core/assets/auth/bootstrap-kubeconfig /etc/kubernetes/
-    # Conformance scripts run kubectl on the master node via admin-kubeconfig
-    cp /home/core/assets/auth/admin-kubeconfig /etc/kubernetes/
+    cp /home/core/assets/auth/kubeconfig /etc/kubernetes/
 
     # Start the kubelet
     systemctl enable kubelet; sudo systemctl start kubelet
@@ -73,32 +71,31 @@ function init_master_node() {
     exit 1
 }
 
-# This script can execute on a remote host by copying itself and the kubelet service unit to remote host.
+# This script can execute on a remote host by copying itself + kubelet service unit to remote host.
 # After assets are available on the remote host, the script will execute itself in "local" mode.
 if [ "${REMOTE_HOST}" != "local" ]; then
     # Set up the kubelet.service on remote host
     scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} kubelet.master core@${REMOTE_HOST}:/home/core/kubelet.master
     ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo mv /home/core/kubelet.master /etc/systemd/system/kubelet.service"
 
-    # Copy self to remote host and execute script in "local" mode
+    # Copy self to remote host so script can be executed in "local" mode
     scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} core@${REMOTE_HOST}:/home/core/init-master.sh
     ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo BOOTKUBE_REPO=${BOOTKUBE_REPO} BOOTKUBE_VERSION=${BOOTKUBE_VERSION} /home/core/init-master.sh local"
 
     # Copy assets from remote host to a local directory. These can be used to launch additional nodes & contain TLS assets
     mkdir ${CLUSTER_DIR}
     scp -q -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} -r core@${REMOTE_HOST}:/home/core/assets/* ${CLUSTER_DIR}
-    sed -i.private "s/server: .*/server: https:\/\/${REMOTE_HOST}:443\//" ${CLUSTER_DIR}/auth/admin-kubeconfig
 
     # Cleanup
-    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo rm -rf /home/core/assets /home/core/init-master.sh"
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "rm -rf /home/core/assets && rm -rf /home/core/init-master.sh"
 
     echo "Cluster assets copied to ${CLUSTER_DIR}"
     echo
     echo "Bootstrap complete. Access your kubernetes cluster using:"
-    echo "kubectl --kubeconfig=${CLUSTER_DIR}/auth/admin-kubeconfig get nodes"
+    echo "kubectl --kubeconfig=${CLUSTER_DIR}/auth/kubeconfig get nodes"
     echo
     echo "Additional nodes can be added to the cluster using:"
-    echo "./init-worker.sh <node-ip>"
+    echo "./init-worker.sh <node-ip> ${CLUSTER_DIR}/auth/kubeconfig"
     echo
 
 # Execute this script locally on the machine, assumes a kubelet.service file has already been placed on host.

hack/quickstart/init-master.sh

@@ -2,8 +2,8 @@
 set -euo pipefail
 
 REMOTE_HOST=$1
+KUBECONFIG=$2
 REMOTE_PORT=${REMOTE_PORT:-22}
-CLUSTER_DIR=${CLUSTER_DIR:-cluster}
 IDENT=${IDENT:-${HOME}/.ssh/id_rsa}
 SSH_OPTS=${SSH_OPTS:-}
 
@@ -13,41 +13,57 @@ function usage() {
     exit 1
 }
 
+function extract_master_endpoint (){
+    grep 'certificate-authority-data' ${KUBECONFIG} | awk '{print $2}' | base64 -d > /home/core/ca.crt
+    grep 'client-certificate-data' ${KUBECONFIG} | awk '{print $2}'| base64 -d > /home/core/client.crt
+    grep 'client-key-data' ${KUBECONFIG} | awk '{print $2}'| base64 -d > /home/core/client.key
+
+    MASTER_PUB="$(awk '/server:/ {print $2}' ${KUBECONFIG} | awk -F/ '{print $3}' | awk -F: '{print $1}')"
+    # TODO (aaron): The -k was added with the gce conformance tests - figure out why it's needed here.
+    #     The certs are seemingly signed correctly, but says no SAN for MASTER_PUB
+    MASTER_PRIV=$(curl -k https://${MASTER_PUB}:443/api/v1/namespaces/default/endpoints/kubernetes \
+        --cacert /home/core/ca.crt --cert /home/core/client.crt --key /home/core/client.key \
+        | jq -r '.subsets[0].addresses[0].ip')
+    rm -f /home/core/ca.crt /home/core/client.crt /home/core/client.key
+}
+
 # Initialize a worker node
 function init_worker_node() {
-    # Setup bootstrap-kubeconfig
+    extract_master_endpoint
+
+    # Setup kubeconfig
     mkdir -p /etc/kubernetes
-    mv /home/core/bootstrap-kubeconfig /etc/kubernetes/bootstrap-kubeconfig
+    cp ${KUBECONFIG} /etc/kubernetes/kubeconfig
 
-    # Move kubelet service file
-    mv /home/core/kubelet.worker /etc/systemd/system/kubelet.service
+    sed "s/{{apiserver}}/${MASTER_PRIV}/" /home/core/kubelet.worker > /etc/systemd/system/kubelet.service
+    rm /home/core/kubelet.worker
 
     # Start services
     systemctl daemon-reload
     systemctl stop update-engine; systemctl mask update-engine
     systemctl enable kubelet; sudo systemctl start kubelet
 }
 
-[ "$#" == 1 ] || usage
+[ "$#" == 2 ] || usage
 
 # This script can execute on a remote host by copying itself + kubelet service unit to remote host.
 # After assets are available on the remote host, the script will execute itself in "local" mode.
 if [ "${REMOTE_HOST}" != "local" ]; then
 
-    # Copy kubelet service file and bootstrap-kubeconfig to remote host
+    # Copy kubelet service file and kubeconfig to remote host
     scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} kubelet.worker core@${REMOTE_HOST}:/home/core/kubelet.worker
-    scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${CLUSTER_DIR}/auth/bootstrap-kubeconfig core@${REMOTE_HOST}:/home/core/bootstrap-kubeconfig
+    scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${KUBECONFIG} core@${REMOTE_HOST}:/home/core/kubeconfig
 
     # Copy self to remote host so script can be executed in "local" mode
     scp -i ${IDENT} -P ${REMOTE_PORT} ${SSH_OPTS} ${BASH_SOURCE[0]} core@${REMOTE_HOST}:/home/core/init-worker.sh
-    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo /home/core/init-worker.sh local"
+    ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "sudo /home/core/init-worker.sh local /home/core/kubeconfig"
 
     # Cleanup
     ssh -i ${IDENT} -p ${REMOTE_PORT} ${SSH_OPTS} core@${REMOTE_HOST} "rm /home/core/init-worker.sh"
 
     echo
     echo "Node bootstrap complete. It may take a few minutes for the node to become ready. Access your kubernetes cluster using:"
-    echo "kubectl --kubeconfig=${CLUSTER_DIR}/auth/admin-kubeconfig get nodes"
+    echo "kubectl --kubeconfig=${KUBECONFIG} get nodes"
     echo
 
 # Execute this script locally on the machine, assumes a kubelet.service file has already been placed on host.

hack/quickstart/init-worker.sh

@@ -11,10 +11,8 @@ ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
 ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
 ExecStartPre=/bin/mkdir -p /var/lib/cni
 ExecStart=/usr/lib/coreos/kubelet-wrapper \
+  --api-servers=https://${COREOS_PRIVATE_IPV4}:443 \
   --kubeconfig=/etc/kubernetes/kubeconfig \
-  --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig \
-  --cert-dir=/etc/kubernetes/secrets \
-  --require-kubeconfig \
   --cni-conf-dir=/etc/kubernetes/cni/net.d \
   --network-plugin=cni \
   --lock-file=/var/run/lock/kubelet.lock \

hack/quickstart/kubelet.master

@@ -9,10 +9,8 @@ ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
 ExecStartPre=/bin/mkdir -p /var/lib/cni
 ExecStart=/usr/lib/coreos/kubelet-wrapper \
+  --api-servers=https://{{apiserver}}:443 \
   --kubeconfig=/etc/kubernetes/kubeconfig \
-  --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubeconfig \
-  --cert-dir=/etc/kubernetes/secrets \
-  --require-kubeconfig \
   --cni-conf-dir=/etc/kubernetes/cni/net.d \
   --network-plugin=cni \
   --lock-file=/var/run/lock/kubelet.lock \

hack/quickstart/kubelet.worker

@@ -60,44 +60,20 @@ $ IDENT=k8s-key.pem ./init-master.sh <PUBLIC_IP>
 After the master bootstrap is complete, you can continue to add worker nodes. Or cluster state can be inspected via kubectl:
 
 ```
-$ kubectl --kubeconfig=cluster/auth/admin-kubeconfig get nodes
+$ kubectl --kubeconfig=cluster/auth/kubeconfig get nodes
 ```
 
 ### Add Workers
 
 Run the `Launch Nodes` step for each additional node you wish to add, then using the public-ip, run:
 
 ```
-IDENT=k8s-key.pem ./init-worker.sh <PUBLIC_IP>
+IDENT=k8s-key.pem ./init-worker.sh <PUBLIC_IP> cluster/auth/kubeconfig
 ```
 
-After a few minutes, time for the required assets and containers to be 
-downloaded, the new worker will submit a Certificate Signing Request. This 
-request must be approved for the worker to join the cluster. Until Kubernetes 
-1.6, there is no [approve/deny] commands built in _kubectl_, therefore we must 
-interact directly with the Kubernetes API. In the example below, we demonstrate
-how the provided [csrctl.sh] tool can be used to manage CSRs.
+**NOTE:** It can take a few minutes for each node to download all of the required assets / containers.
+ They may not be immediately available, but the state can be inspected with:
 
 ```
-$ ../csrctl.sh cluster/auth/admin-kubeconfig list
-NAME        AGE       REQUESTOR           CONDITION
-csr-9fxjw   16m       kubelet-bootstrap   Pending
-csr-j9r05   22m       kubelet-bootstrap   Approved,Issued
-
-$ ../csrctl.sh cluster/auth/admin-kubeconfig get csr-9fxjw
-$ ../csrctl.sh cluster/auth/admin-kubeconfig approve csr-9fxjw
-
-$ ../csrctl.sh cluster/auth/admin-kubeconfig list
-NAME        AGE       REQUESTOR           CONDITION
-csr-9fxjw   16m       kubelet-bootstrap   Approved,Issued
-csr-j9r05   22m       kubelet-bootstrap   Approved,Issued
+$ kubectl --kubeconfig=cluster/auth/kubeconfig get nodes
 ```
-
-Once approved, the worker node should appear immediately in the node list:
-
-```
-$ kubectl --kubeconfig=cluster/auth/admin-kubeconfig get nodes
-```
-
-[approve/deny]: https://github.com/kubernetes/kubernetes/issues/30163
-[csrctl.sh]: ../csrctl.sh

hack/quickstart/quickstart-aws.md

@@ -36,7 +36,7 @@ $ IDENT=~/.ssh/google_compute_engine ./init-master.sh <node-ip>
 After the master bootstrap is complete, you can continue to add worker nodes. Or cluster state can be inspected via kubectl:
 
 ```
-$ kubectl --kubeconfig=cluster/auth/admin-kubeconfig get nodes
+$ kubectl --kubeconfig=cluster/auth/kubeconfig get nodes
 ```
 
 ### Add Workers
@@ -53,36 +53,12 @@ $ gcloud compute instances list ${CLUSTER_PREFIX}-core3
 Initialize each worker node by replacing `<node-ip>` with the EXTERNAL_IP from the commands above.
 
 ```
-$ IDENT=~/.ssh/google_compute_engine ./init-worker.sh <node-ip>
+$ IDENT=~/.ssh/google_compute_engine ./init-worker.sh <node-ip> cluster/auth/kubeconfig
 ```
 
-After a few minutes, time for the required assets and containers to be 
-downloaded, the new worker will submit a Certificate Signing Request. This 
-request must be approved for the worker to join the cluster. Until Kubernetes 
-1.6, there is no [approve/deny] commands built in _kubectl_, therefore we must 
-interact directly with the Kubernetes API. In the example below, we demonstrate
-how the provided [csrctl.sh] tool can be used to manage CSRs.
+**NOTE:** It can take a few minutes for each node to download all of the required assets / containers.
+ They may not be immediately available, but the state can be inspected with:
 
 ```
-$ ../csrctl.sh cluster/auth/admin-kubeconfig list
-NAME        AGE       REQUESTOR           CONDITION
-csr-9fxjw   16m       kubelet-bootstrap   Pending
-csr-j9r05   22m       kubelet-bootstrap   Approved,Issued
-
-$ ../csrctl.sh cluster/auth/admin-kubeconfig get csr-9fxjw
-$ ../csrctl.sh cluster/auth/admin-kubeconfig approve csr-9fxjw
-
-$ ../csrctl.sh cluster/auth/admin-kubeconfig list
-NAME        AGE       REQUESTOR           CONDITION
-csr-9fxjw   16m       kubelet-bootstrap   Approved,Issued
-csr-j9r05   22m       kubelet-bootstrap   Approved,Issued
+$ kubectl --kubeconfig=cluster/auth/kubeconfig get nodes
 ```
-
-Once approved, the worker node should appear immediately in the node list:
-
-```
-$ kubectl --kubeconfig=cluster/auth/admin-kubeconfig get nodes
-```
-
-[approve/deny]: https://github.com/kubernetes/kubernetes/issues/30163
-[csrctl.sh]: ../csrctl.sh