bootstrap: Use kubeconfig contents as seed for cert dir if necessary

kubeadm uses certificate rotation to replace the initial high-power
cert provided in --kubeconfig with a less powerful certificate on
the masters. This requires that we pass the contents of the client
config certData and keyData down into the cert store to populate
the initial client.

Add better comments to describe why the flow is required. Add a test
that verifies initial cert contents are written to disk. Change
the cert manager to not use MustRegister for prometheus so that
it can be tested.

Author: smarterclayton

cmd/kubelet/app/server.go

@@ -739,6 +739,24 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
 // bootstrapping is enabled or client certificate rotation is enabled.
 func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) (*restclient.Config, func(), error) {
 	if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
+		// Rules for client rotation and the handling of kube config files:
+		//
+		// 1. If the client provides only a kubeconfig file, we must use that as the initial client
+		//    kubeadm needs the initial data in the kubeconfig to be placed into the cert store
+		// 2. If the client provides only an initial bootstrap kubeconfig file, we must create a
+		//    kubeconfig file at the target location that points to the cert store, but until
+		//    the file is present the client config will have no certs
+		// 3. If the client provides both and the kubeconfig is valid, we must ignore the bootstrap
+		//    kubeconfig.
+		// 4. If the client provides both and the kubeconfig is expired or otherwise invalid, we must
+		//    replace the kubeconfig with a new file that points to the cert dir
+		//
+		// The desired configuration for bootstrapping is to use a bootstrap kubeconfig and to have
+		// the kubeconfig file be managed by this process. For backwards compatibility with kubeadm,
+		// which provides a high powered kubeconfig on the master with cert/key data, we must
+		// bootstrap the cert manager with the contents of the initial client config.
+
+		klog.Infof("Client rotation is on, will bootstrap in background")
 		certConfig, clientConfig, err := bootstrap.LoadClientConfig(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory)
 		if err != nil {
 			return nil, nil, err
@@ -750,9 +768,8 @@ func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName)
 		}
 
 		// the rotating transport will use the cert from the cert manager instead of these files
-		transportConfig := restclient.CopyConfig(clientConfig)
-		transportConfig.CertFile = ""
-		transportConfig.KeyFile = ""
+		transportConfig := restclient.AnonymousClientConfig(clientConfig)
+		kubeClientConfigOverrides(s, transportConfig)
 
 		// we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable
 		// to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper
@@ -774,10 +791,16 @@ func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName)
 		}
 	}
 
-	clientConfig, err := createAPIServerClientConfig(s)
+	clientConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig},
+		&clientcmd.ConfigOverrides{},
+	).ClientConfig()
 	if err != nil {
 		return nil, nil, fmt.Errorf("invalid kubeconfig: %v", err)
 	}
+
+	kubeClientConfigOverrides(s, clientConfig)
+
 	return clientConfig, nil, nil
 }
 
@@ -804,12 +827,26 @@ func buildClientCertificateManager(certConfig, clientConfig *restclient.Config,
 	return kubeletcertificate.NewKubeletClientCertificateManager(
 		certDir,
 		nodeName,
+
+		// this preserves backwards compatibility with kubeadm which passes
+		// a high powered certificate to the kubelet as --kubeconfig and expects
+		// it to be rotated out immediately
+		clientConfig.CertData,
+		clientConfig.KeyData,
+
 		clientConfig.CertFile,
 		clientConfig.KeyFile,
 		newClientFn,
 	)
 }
 
+func kubeClientConfigOverrides(s *options.KubeletServer, clientConfig *restclient.Config) {
+	clientConfig.ContentType = s.ContentType
+	// Override kubeconfig qps/burst settings from flags
+	clientConfig.QPS = float32(s.KubeAPIQPS)
+	clientConfig.Burst = int(s.KubeAPIBurst)
+}
+
 // getNodeName returns the node name according to the cloud provider
 // if cloud provider is specified. Otherwise, returns the hostname of the node.
 func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) {
@@ -898,38 +935,6 @@ func InitializeTLS(kf *options.KubeletFlags, kc *kubeletconfiginternal.KubeletCo
 	return tlsOptions, nil
 }
 
-func kubeconfigClientConfig(s *options.KubeletServer) (*restclient.Config, error) {
-	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
-		&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.KubeConfig},
-		&clientcmd.ConfigOverrides{},
-	).ClientConfig()
-}
-
-// createClientConfig creates a client configuration from the command line arguments.
-// If --kubeconfig is explicitly set, it will be used.
-func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) {
-	if len(s.BootstrapKubeconfig) > 0 || len(s.KubeConfig) > 0 {
-		return kubeconfigClientConfig(s)
-	}
-	return nil, fmt.Errorf("createClientConfig called in standalone mode")
-}
-
-// createAPIServerClientConfig generates a client.Config from command line flags
-// via createClientConfig and then injects chaos into the configuration via addChaosToClientConfig.
-func createAPIServerClientConfig(s *options.KubeletServer) (*restclient.Config, error) {
-	clientConfig, err := createClientConfig(s)
-	if err != nil {
-		return nil, err
-	}
-
-	clientConfig.ContentType = s.ContentType
-	// Override kubeconfig qps/burst settings from flags
-	clientConfig.QPS = float32(s.KubeAPIQPS)
-	clientConfig.Burst = int(s.KubeAPIBurst)
-
-	return clientConfig, nil
-}
-
 // RunKubelet is responsible for setting up and running a kubelet.  It is used in three different applications:
 //   1 Integration tests
 //   2 Kubelet binary

cmd/kubelet/app/server_bootstrap_test.go

@@ -19,10 +19,13 @@ package app
 import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
-	cryptorand "crypto/rand"
+	"crypto/rand"
 	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/json"
+	"encoding/pem"
 	"io/ioutil"
+	"math/big"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -53,7 +56,7 @@ func Test_buildClientCertificateManager(t *testing.T) {
 	}
 	defer func() { os.RemoveAll(testDir) }()
 
-	serverPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), cryptorand.Reader)
+	serverPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -132,6 +135,65 @@ func Test_buildClientCertificateManager(t *testing.T) {
 	}
 }
 
+func Test_buildClientCertificateManager_populateCertDir(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "kubeletcert")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { os.RemoveAll(testDir) }()
+
+	// when no cert is provided, write nothing to disk
+	config1 := &restclient.Config{
+		UserAgent: "FirstClient",
+		Host:      "http://localhost",
+	}
+	config2 := &restclient.Config{
+		UserAgent: "SecondClient",
+		Host:      "http://localhost",
+	}
+	nodeName := types.NodeName("test")
+	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
+		t.Fatal(err)
+	}
+	fi := getFileInfo(testDir)
+	if len(fi) != 0 {
+		t.Fatalf("Unexpected directory contents: %#v", fi)
+	}
+
+	// an invalid cert should be ignored
+	config2.CertData = []byte("invalid contents")
+	config2.KeyData = []byte("invalid contents")
+	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err == nil {
+		t.Fatal("unexpected non error")
+	}
+	fi = getFileInfo(testDir)
+	if len(fi) != 0 {
+		t.Fatalf("Unexpected directory contents: %#v", fi)
+	}
+
+	// an expired client certificate should be written to disk, because the cert manager can
+	// use config1 to refresh it and the cert manager won't return it for clients.
+	config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-2*time.Hour), time.Now().Add(-time.Hour))
+	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
+		t.Fatal(err)
+	}
+	fi = getFileInfo(testDir)
+	if len(fi) != 2 {
+		t.Fatalf("Unexpected directory contents: %#v", fi)
+	}
+
+	// a valid, non-expired client certificate should be written to disk
+	config2.CertData, config2.KeyData = genClientCert(t, time.Now().Add(-time.Hour), time.Now().Add(24*time.Hour))
+	if _, err := buildClientCertificateManager(config1, config2, testDir, nodeName); err != nil {
+		t.Fatal(err)
+	}
+	fi = getFileInfo(testDir)
+	if len(fi) != 2 {
+		t.Fatalf("Unexpected directory contents: %#v", fi)
+	}
+
+}
+
 func getFileInfo(dir string) map[string]os.FileInfo {
 	fi := make(map[string]os.FileInfo)
 	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
@@ -279,3 +341,37 @@ func (s *csrSimulator) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
 	}
 }
+
+// genClientCert generates an x509 certificate for testing. Certificate and key
+// are returned in PEM encoding.
+func genClientCert(t *testing.T, from, to time.Time) ([]byte, []byte) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	keyRaw, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cert := &x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject:      pkix.Name{Organization: []string{"Acme Co"}},
+		NotBefore:    from,
+		NotAfter:     to,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+	}
+	certRaw, err := x509.CreateCertificate(rand.Reader, cert, cert, key.Public(), key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certRaw}),
+		pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyRaw})
+}

pkg/kubelet/certificate/bootstrap/bootstrap.go

@@ -59,15 +59,16 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err)
 		}
-		return clientConfig, clientConfig, nil
+		klog.V(2).Infof("No bootstrapping requested, will use kubeconfig")
+		return clientConfig, restclient.CopyConfig(clientConfig), nil
 	}
 
 	store, err := certificate.NewFileStore("kubelet-client", certDir, certDir, "", "")
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to build bootstrap cert store")
 	}
 
-	ok, err := verifyBootstrapClientConfig(kubeconfigPath)
+	ok, err := isClientConfigStillValid(kubeconfigPath)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -78,7 +79,8 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to load kubeconfig: %v", err)
 		}
-		return clientConfig, clientConfig, nil
+		klog.V(2).Infof("Current kubeconfig file contents are still valid, no bootstrap necessary")
+		return clientConfig, restclient.CopyConfig(clientConfig), nil
 	}
 
 	bootstrapClientConfig, err := loadRESTClientConfig(bootstrapPath)
@@ -93,6 +95,7 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig
 	if err := writeKubeconfigFromBootstrapping(clientConfig, kubeconfigPath, pemPath); err != nil {
 		return nil, nil, err
 	}
+	klog.V(2).Infof("Use the bootstrap credentials to request a cert, and set kubeconfig to point to the certificate dir")
 	return bootstrapClientConfig, clientConfig, nil
 }
 
@@ -102,7 +105,7 @@ func LoadClientConfig(kubeconfigPath, bootstrapPath, certDir string) (certConfig
 // The certificate and key file are stored in certDir.
 func LoadClientCert(kubeconfigPath, bootstrapPath, certDir string, nodeName types.NodeName) error {
 	// Short-circuit if the kubeconfig file exists and is valid.
-	ok, err := verifyBootstrapClientConfig(kubeconfigPath)
+	ok, err := isClientConfigStillValid(kubeconfigPath)
 	if err != nil {
 		return err
 	}
@@ -219,10 +222,10 @@ func loadRESTClientConfig(kubeconfig string) (*restclient.Config, error) {
 	).ClientConfig()
 }
 
-// verifyBootstrapClientConfig checks the provided kubeconfig to see if it has a valid
+// isClientConfigStillValid checks the provided kubeconfig to see if it has a valid
 // client certificate. It returns true if the kubeconfig is valid, or an error if bootstrapping
 // should stop immediately.
-func verifyBootstrapClientConfig(kubeconfigPath string) (bool, error) {
+func isClientConfigStillValid(kubeconfigPath string) (bool, error) {
 	_, err := os.Stat(kubeconfigPath)
 	if os.IsNotExist(err) {
 		return false, nil

pkg/kubelet/certificate/kubelet.go

@@ -147,7 +147,16 @@ func addressesToHostnamesAndIPs(addresses []v1.NodeAddress) (dnsNames []string,
 // NewKubeletClientCertificateManager sets up a certificate manager without a
 // client that can be used to sign new certificates (or rotate). If a CSR
 // client is set later, it may begin rotating/renewing the client cert.
-func NewKubeletClientCertificateManager(certDirectory string, nodeName types.NodeName, certFile string, keyFile string, clientFn certificate.CSRClientFunc) (certificate.Manager, error) {
+func NewKubeletClientCertificateManager(
+	certDirectory string,
+	nodeName types.NodeName,
+	bootstrapCertData []byte,
+	bootstrapKeyData []byte,
+	certFile string,
+	keyFile string,
+	clientFn certificate.CSRClientFunc,
+) (certificate.Manager, error) {
+
 	certificateStore, err := certificate.NewFileStore(
 		"kubelet-client",
 		certDirectory,
@@ -165,7 +174,7 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
 			Help:      "Gauge of the lifetime of a certificate. The value is the date the certificate will expire in seconds since January 1, 1970 UTC.",
 		},
 	)
-	prometheus.MustRegister(certificateExpiration)
+	prometheus.Register(certificateExpiration)
 
 	m, err := certificate.NewManager(&certificate.Config{
 		ClientFn: clientFn,
@@ -190,6 +199,14 @@ func NewKubeletClientCertificateManager(certDirectory string, nodeName types.Nod
 			// authenticate itself to the TLS server.
 			certificates.UsageClientAuth,
 		},
+
+		// For backwards compatibility, the kubelet supports the ability to
+		// provide a higher privileged certificate as initial data that will
+		// then be rotated immediately. This code path is used by kubeadm on
+		// the masters.
+		BootstrapCertificatePEM: bootstrapCertData,
+		BootstrapKeyPEM:         bootstrapKeyData,
+
 		CertificateStore:      certificateStore,
 		CertificateExpiration: certificateExpiration,
 	})