When using external CA, look for common trust anchor within CA bundle.

astundzia · astundzia · commit 58ff17bbfae6 · 2024-06-28T18:04:31.000-05:00
diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig.go
@@ -264,10 +264,36 @@ func validateKubeConfig(outDir, filename string, config *clientcmdapi.Config) er
 	}
 	caExpected := bytes.TrimSpace(config.Clusters[expectedCluster].CertificateAuthorityData)
 
-	// If the current CA cert on disk doesn't match the expected CA cert, error out because we have a file, but it's stale
-	if !bytes.Equal(caCurrent, caExpected) {
-		return errors.Errorf("a kubeconfig file %q exists already but has got the wrong CA cert", kubeConfigFilePath)
+	// Parse the current certificate authority data
+	currentCACerts, err := certutil.ParseCertsPEM(caCurrent)
+	if err != nil {
+		return errors.Errorf("the kubeconfig file %q contains an invalid CA cert", kubeConfigFilePath)
+	}
+
+	// Parse the expected certificate authority data
+	expectedCACerts, err := certutil.ParseCertsPEM(caExpected)
+	if err != nil {
+		return errors.Errorf("the expected base64 encoded CA cert could not be parsed as a PEM:\n%s\n", caExpected)
+	}
+
+	// Only use the first certificate in the current CA cert list
+	currentCaCert := currentCACerts[0]
+
+	// Find a common trust anchor
+	trustAnchorFound := false
+	for _, expectedCaCert := range expectedCACerts {
+		// Compare the current CA cert to the expected CA cert.
+		// If the certificates match then a common trust anchor was found.
+		if currentCaCert.Equal(expectedCaCert) {
+			trustAnchorFound = true
+			break
+		}
+	}
+	if !trustAnchorFound {
+		return errors.Errorf("a kubeconfig file %q exists but does not contain a trusted CA in its current context's "+
+			"cluster. Total CA certificates found: %d", kubeConfigFilePath, len(currentCACerts))
 	}
+
 	// If the current API Server location on disk doesn't match the expected API server, show a warning
 	if currentConfig.Clusters[currentCluster].Server != config.Clusters[expectedCluster].Server {
 		klog.Warningf("a kubeconfig file %q exists already but has an unexpected API Server URL: expected: %s, got: %s",
@@ -386,20 +412,31 @@ func writeKubeConfigFromSpec(out io.Writer, spec *kubeConfigSpec, clustername st
 func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfiguration) error {
 	// Creates a kubeconfig file with the target CA and server URL
 	// to be used as a input for validating user provided kubeconfig files
-	caCert, err := pkiutil.TryLoadCertFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
+	caCert, intermediaries, err := pkiutil.TryLoadCertChainFromDisk(cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
 	if err != nil {
 		return errors.Wrapf(err, "the CA file couldn't be loaded")
 	}
+
+	// Combine caCert and intermediaries into one array
+	caCertChain := append([]*x509.Certificate{caCert}, intermediaries...)
+
 	// Validate period
-	certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, caCert)
+	for _, cert := range caCertChain {
+		certsphase.CheckCertificatePeriodValidity(kubeadmconstants.CACertAndKeyBaseName, cert)
+	}
 
 	// validate user provided kubeconfig files for the scheduler and controller-manager
 	localAPIEndpoint, err := kubeadmutil.GetLocalAPIEndpoint(&cfg.LocalAPIEndpoint)
 	if err != nil {
 		return err
 	}
 
-	validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
+	caCertBytes, err := pkiutil.EncodeCertBundlePEM(caCertChain)
+	if err != nil {
+		return err
+	}
+
+	validationConfigLocal := kubeconfigutil.CreateBasic(localAPIEndpoint, "dummy", "dummy", caCertBytes)
 	kubeConfigFileNamesLocal := []string{
 		kubeadmconstants.ControllerManagerKubeConfigFileName,
 		kubeadmconstants.SchedulerKubeConfigFileName,
@@ -417,7 +454,7 @@ func ValidateKubeconfigsForExternalCA(outDir string, cfg *kubeadmapi.InitConfigu
 		return err
 	}
 
-	validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", pkiutil.EncodeCertPEM(caCert))
+	validationConfigCPE := kubeconfigutil.CreateBasic(controlPlaneEndpoint, "dummy", "dummy", caCertBytes)
 	kubeConfigFileNamesCPE := []string{
 		kubeadmconstants.AdminKubeConfigFileName,
 		kubeadmconstants.SuperAdminKubeConfigFileName,
diff --git a/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go b/cmd/kubeadm/app/phases/kubeconfig/kubeconfig_test.go
@@ -608,7 +608,11 @@ func TestValidateKubeConfig(t *testing.T) {
 
 func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 	tmpDir := testutil.SetupTempDir(t)
-	defer os.RemoveAll(tmpDir)
+	defer func() {
+		if err := os.RemoveAll(tmpDir); err != nil {
+			t.Error(err)
+		}
+	}()
 	pkiDir := filepath.Join(tmpDir, "pki")
 
 	initConfig := &kubeadmapi.InitConfiguration{
@@ -623,11 +627,9 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 
 	// creates CA, write to pkiDir and remove ca.key to get into external CA condition
 	caCert, caKey := certstestutil.SetupCertificateAuthority(t)
-	if err := pkiutil.WriteCertAndKey(pkiDir, kubeadmconstants.CACertAndKeyBaseName, caCert, caKey); err != nil {
-		t.Fatalf("failure while saving CA certificate and key: %v", err)
-	}
-	if err := os.Remove(filepath.Join(pkiDir, kubeadmconstants.CAKeyName)); err != nil {
-		t.Fatalf("failure while deleting ca.key: %v", err)
+
+	if err := pkiutil.WriteCertBundle(pkiDir, kubeadmconstants.CACertAndKeyBaseName, []*x509.Certificate{caCert}); err != nil {
+		t.Fatalf("failure while saving CA certificate: %v", err)
 	}
 
 	notAfter, _ := time.Parse(time.RFC3339, "2026-01-02T15:04:05Z")
@@ -697,7 +699,11 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
 			tmpdir := testutil.SetupTempDir(t)
-			defer os.RemoveAll(tmpdir)
+			defer func() {
+				if err := os.RemoveAll(tmpdir); err != nil {
+					t.Error(err)
+				}
+			}()
 
 			for name, config := range test.filesToWrite {
 				if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil {
@@ -719,6 +725,166 @@ func TestValidateKubeconfigsForExternalCA(t *testing.T) {
 	}
 }
 
+func TestValidateKubeconfigsForExternalCAMissingRoot(t *testing.T) {
+	tmpDir := testutil.SetupTempDir(t)
+	defer func() {
+		if err := os.RemoveAll(tmpDir); err != nil {
+			t.Error(err)
+		}
+	}()
+	pkiDir := filepath.Join(tmpDir, "pki")
+
+	initConfig := &kubeadmapi.InitConfiguration{
+		ClusterConfiguration: kubeadmapi.ClusterConfiguration{
+			CertificatesDir: pkiDir,
+		},
+		LocalAPIEndpoint: kubeadmapi.APIEndpoint{
+			BindPort:         1234,
+			AdvertiseAddress: "1.2.3.4",
+		},
+	}
+
+	// Creates CA, write to pkiDir and remove ca.key to get into external CA mode
+	caCert, caKey := certstestutil.SetupCertificateAuthority(t)
+
+	// Setup multiple intermediate certificate authorities (CAs) for testing purposes.
+	// This is "Root CA" signs "Intermediate Authority 1A" signs "Intermediate Authority 2A"
+	intermediateCACert1a, intermediateCAKey1a := certstestutil.SetupIntermediateCertificateAuthority(t, caCert, caKey, "Intermediate Authority 1A")
+	intermediateCACert2a, intermediateCAKey2a := certstestutil.SetupIntermediateCertificateAuthority(t, intermediateCACert1a, intermediateCAKey1a, "Intermediate Authority 1A")
+
+	// These two CA certificates should both validate using the Intermediate CA 2B certificate
+	// This is "Root CA" signs "Intermediate Authority 1B" signs "Intermediate Authority 2B"
+	intermediateCACert1b, intermediateCAKey1b := certstestutil.SetupIntermediateCertificateAuthority(t, caCert, caKey, "Intermediate Authority 1B")
+	intermediateCACert2b, intermediateCAKey2b := certstestutil.SetupIntermediateCertificateAuthority(t, intermediateCACert1b, intermediateCAKey1b, "Intermediate Authority 2B")
+
+	notAfter, _ := time.Parse(time.RFC3339, "2036-01-02T15:04:05Z")
+	clusterName := "myOrg1"
+
+	var validCaCertBundle []*x509.Certificate
+	validCaCertBundle = append(validCaCertBundle, caCert, intermediateCACert1a, intermediateCACert2a)
+	multipleCAConfigRootCAIssuer := setupKubeConfigWithClientAuth(t, caCert, caKey, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
+	multipleCAConfigIntermediateCA1aIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert1a, intermediateCAKey1a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
+	multipleCAConfigIntermediateCA2aIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert2a, intermediateCAKey2a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
+
+	var caBundleMissingRootCA []*x509.Certificate
+	caBundleMissingRootCA = append(caBundleMissingRootCA, intermediateCACert1b, intermediateCACert2b)
+	multipleCAConfigNoRootCA := setupKubeConfigWithClientAuth(t, intermediateCACert2b, intermediateCAKey2b, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
+	multipleCAConfigDifferentIssuer := setupKubeConfigWithClientAuth(t, intermediateCACert2a, intermediateCAKey2a, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
+
+	var caBundlePartialChain []*x509.Certificate
+	caBundlePartialChain = append(caBundlePartialChain, intermediateCACert1a)
+	multipleCaPartialCA := setupKubeConfigWithClientAuth(t, intermediateCACert2b, intermediateCAKey2b, notAfter, "https://1.2.3.4:1234", "test-cluster", clusterName)
+
+	tests := map[string]struct {
+		filesToWrite  map[string]*clientcmdapi.Config
+		initConfig    *kubeadmapi.InitConfiguration
+		expectedError bool
+		caCertificate []*x509.Certificate
+	}{
+		// Positive test cases
+		"valid config issued from RootCA": {
+			filesToWrite: map[string]*clientcmdapi.Config{
+				kubeadmconstants.AdminKubeConfigFileName:             multipleCAConfigRootCAIssuer,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        multipleCAConfigRootCAIssuer,
+				kubeadmconstants.KubeletKubeConfigFileName:           multipleCAConfigRootCAIssuer,
+				kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigRootCAIssuer,
+				kubeadmconstants.SchedulerKubeConfigFileName:         multipleCAConfigRootCAIssuer,
+			},
+			caCertificate: validCaCertBundle,
+			initConfig:    initConfig,
+			expectedError: false,
+		},
+		"valid config issued from IntermediateCA 1A": {
+			filesToWrite: map[string]*clientcmdapi.Config{
+				kubeadmconstants.AdminKubeConfigFileName:             multipleCAConfigIntermediateCA1aIssuer,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        multipleCAConfigIntermediateCA1aIssuer,
+				kubeadmconstants.KubeletKubeConfigFileName:           multipleCAConfigIntermediateCA1aIssuer,
+				kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigIntermediateCA1aIssuer,
+				kubeadmconstants.SchedulerKubeConfigFileName:         multipleCAConfigIntermediateCA1aIssuer,
+			},
+			caCertificate: validCaCertBundle,
+			initConfig:    initConfig,
+			expectedError: false,
+		},
+		"valid config issued from IntermediateCA 2A": {
+			filesToWrite: map[string]*clientcmdapi.Config{
+				kubeadmconstants.AdminKubeConfigFileName:             multipleCAConfigIntermediateCA2aIssuer,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        multipleCAConfigIntermediateCA2aIssuer,
+				kubeadmconstants.KubeletKubeConfigFileName:           multipleCAConfigIntermediateCA2aIssuer,
+				kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigIntermediateCA2aIssuer,
+				kubeadmconstants.SchedulerKubeConfigFileName:         multipleCAConfigIntermediateCA2aIssuer,
+			},
+			caCertificate: validCaCertBundle,
+			initConfig:    initConfig,
+			expectedError: false,
+		},
+		"valid config issued from IntermediateCA 2B, CA missing root certificate": {
+			filesToWrite: map[string]*clientcmdapi.Config{
+				kubeadmconstants.AdminKubeConfigFileName:             multipleCAConfigNoRootCA,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        multipleCAConfigNoRootCA,
+				kubeadmconstants.KubeletKubeConfigFileName:           multipleCAConfigNoRootCA,
+				kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigNoRootCA,
+				kubeadmconstants.SchedulerKubeConfigFileName:         multipleCAConfigNoRootCA,
+			},
+			caCertificate: caBundleMissingRootCA,
+			initConfig:    initConfig,
+			expectedError: false,
+		},
+		// Negative test cases
+		"invalid config issued from IntermediateCA 2A, testing a chain with a different issuer": {
+			filesToWrite: map[string]*clientcmdapi.Config{
+				kubeadmconstants.AdminKubeConfigFileName:             multipleCAConfigDifferentIssuer,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        multipleCAConfigDifferentIssuer,
+				kubeadmconstants.KubeletKubeConfigFileName:           multipleCAConfigDifferentIssuer,
+				kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCAConfigDifferentIssuer,
+				kubeadmconstants.SchedulerKubeConfigFileName:         multipleCAConfigDifferentIssuer,
+			},
+			caCertificate: caBundleMissingRootCA,
+			initConfig:    initConfig,
+			expectedError: true,
+		},
+		"invalid config issued from IntermediateCA 2B chain, CA only contains Intermediate 1A": {
+			filesToWrite: map[string]*clientcmdapi.Config{
+				kubeadmconstants.AdminKubeConfigFileName:             multipleCaPartialCA,
+				kubeadmconstants.SuperAdminKubeConfigFileName:        multipleCaPartialCA,
+				kubeadmconstants.KubeletKubeConfigFileName:           multipleCaPartialCA,
+				kubeadmconstants.ControllerManagerKubeConfigFileName: multipleCaPartialCA,
+				kubeadmconstants.SchedulerKubeConfigFileName:         multipleCaPartialCA,
+			},
+			caCertificate: caBundlePartialChain,
+			initConfig:    initConfig,
+			expectedError: true,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			tmpdir := testutil.SetupTempDir(t)
+			defer func() {
+				if err := os.RemoveAll(tmpdir); err != nil {
+					t.Error(err)
+				}
+			}()
+
+			for name, config := range test.filesToWrite {
+				if err := createKubeConfigFileIfNotExists(tmpdir, name, config); err != nil {
+					t.Errorf("createKubeConfigFileIfNotExists failed: %v", err)
+				}
+			}
+
+			if err := pkiutil.WriteCertBundle(pkiDir, kubeadmconstants.CACertAndKeyBaseName, test.caCertificate); err != nil {
+				t.Fatalf("Failure while saving CA certificate: %v", err)
+			}
+
+			err := ValidateKubeconfigsForExternalCA(tmpdir, test.initConfig)
+			if (err != nil) != test.expectedError {
+				t.Fatalf("ValidateKubeconfigsForExternalCA failed\n%s\nexpected error: %t\n\tgot: %t\nerror: %v",
+					name, test.expectedError, (err != nil), err)
+			}
+		})
+	}
+}
+
 // setupKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With ClientAuth
 func setupKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey crypto.Signer, notAfter time.Time, apiServer, clientName, clustername string, organizations ...string) *clientcmdapi.Config {
 	spec := &kubeConfigSpec{
@@ -740,7 +906,7 @@ func setupKubeConfigWithClientAuth(t *testing.T, caCert *x509.Certificate, caKey
 	return config
 }
 
-// setupKubeConfigWithClientAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token
+// setupKubeConfigWithTokenAuth is a test utility function that wraps buildKubeConfigFromSpec for building a KubeConfig object With Token
 func setupKubeConfigWithTokenAuth(t *testing.T, caCert *x509.Certificate, apiServer, clientName, token, clustername string) *clientcmdapi.Config {
 	spec := &kubeConfigSpec{
 		CACert:     caCert,
diff --git a/cmd/kubeadm/app/util/certs/util.go b/cmd/kubeadm/app/util/certs/util.go
@@ -38,7 +38,20 @@ func SetupCertificateAuthority(t *testing.T) (*x509.Certificate, crypto.Signer)
 		Config: certutil.Config{CommonName: "kubernetes"},
 	})
 	if err != nil {
-		t.Fatalf("failure while generating CA certificate and key: %v", err)
+		t.Fatalf("Failure while generating CA certificate and key: %v", err)
+	}
+
+	return caCert, caKey
+}
+
+// SetupIntermediateCertificateAuthority is a utility function for kubeadm testing that creates a
+// Intermediate CertificateAuthority cert/key pair
+func SetupIntermediateCertificateAuthority(t *testing.T, parentCert *x509.Certificate, parentKey crypto.Signer, cn string) (*x509.Certificate, crypto.Signer) {
+	caCert, caKey, err := pkiutil.NewIntermediateCertificateAuthority(parentCert, parentKey, &pkiutil.CertConfig{
+		Config: certutil.Config{CommonName: cn},
+	})
+	if err != nil {
+		t.Fatalf("Failure while generating intermediate CA certificate and key: %v", err)
 	}
 
 	return caCert, caKey
