fix: preserve registry addon root CA when moving management cluster

Author: dkoshkin

charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml

@@ -32,20 +32,3 @@ spec:
     kind: {{ .Values.certificates.issuer.kind }}
     name: {{ template "chart.issuerName" . }}
   secretName: {{ template "chart.name" . }}-admission-tls
----
-# CA used to sign certificates for the clusters' registry addons
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: registry-addon-root-ca
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "chart.labels" . | nindent 4 }}
-spec:
-  isCA: true
-  commonName: registry-addon
-  secretName: registry-addon-root-ca
-  issuerRef:
-    kind: {{ .Values.certificates.issuer.kind }}
-    name: {{ template "chart.issuerName" . }}
-  duration: 87600h  # 10 years

common/pkg/capi/utils/utils.go

@@ -15,7 +15,7 @@ import (
 )
 
 // ManagementCluster returns a Cluster object if c is pointing to a management cluster, otherwise returns nil.
-func ManagementCluster(ctx context.Context, c client.Client) (*clusterv1.Cluster, error) {
+func ManagementCluster(ctx context.Context, c client.Reader) (*clusterv1.Cluster, error) {
 	allNodes := &corev1.NodeList{}
 	err := c.List(ctx, allNodes)
 	if err != nil {

pkg/handlers/generic/lifecycle/registry/cncfdistribution/handler.go

@@ -73,8 +73,14 @@ func (n *CNCFDistribution) Setup(
 	cluster *clusterv1.Cluster,
 	log logr.Logger,
 ) error {
+	log.Info("Setting up root CA for CNCF Distribution registry if not already present")
+	err := utils.EnsureRegistryAddonRootCASecret(ctx, n.client, cluster)
+	if err != nil {
+		return fmt.Errorf("failed to ensure root CA secret for CNCF Distribution registry addon: %w", err)
+	}
+
 	log.Info("Setting up CA for CNCF Distribution registry")
-	err := utils.EnsureCASecretForCluster(
+	err = utils.EnsureCASecretForCluster(
 		ctx,
 		n.client,
 		cluster,

pkg/handlers/generic/lifecycle/registry/utils/tls.go

@@ -30,13 +30,105 @@ const (
 )
 
 var (
+	// Valid for 10 years.
+	defaultRootCADuration = 10 * 365 * 24 * time.Hour
+
 	// Similar to CAPI, set the NotBefore to a few minutes in the past to account for clock skew.
 	// This cert is being generated on the management cluster, but used by a workload cluster.
 	defaultCertificateNotBeforeSkew = 5 * time.Minute
 	// Valid for 2 years to avoid expiring before the cluster is upgraded.
 	defaultCertificateDuration = 2 * 365 * 24 * time.Hour
 )
 
+// EnsureRegistryAddonRootCASecret ensures that the registry addon root CA secret exists.
+// This Secret is used to sign the registry TLS certificates for the remote clusters.
+func EnsureRegistryAddonRootCASecret(
+	ctx context.Context,
+	c ctrlclient.Client,
+	managementCluster *clusterv1.Cluster,
+) error {
+	globalTLSCertificateSecret, err := handlersutils.SecretForRegistryAddonRootCA(ctx, c)
+	if err != nil && !handlersutils.IsSecretNotFoundError(err) {
+		return err
+	}
+	// If the secret already exists, we don't need to do anything.
+	if globalTLSCertificateSecret != nil {
+		return nil
+	}
+
+	certPEM, keyPEM, err := generateRegistryAddonRootCAData()
+	if err != nil {
+		return fmt.Errorf("failed to generate registry addon root CA data: %w", err)
+	}
+	rootCASecret := buildRegistryAddonRootCASecret(certPEM, keyPEM, managementCluster)
+
+	err = handlersutils.EnsureSecretForLocalCluster(ctx, c, rootCASecret, managementCluster)
+	if err != nil {
+		return fmt.Errorf("failed to ensure registry addon root CA secret: %w", err)
+	}
+
+	return nil
+}
+
+func generateRegistryAddonRootCAData() (certPEM, keyPEM []byte, err error) {
+	// 1. generate a new RSA private key.
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to generate private key: %w", err)
+	}
+
+	serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to generate serial number: %w", err)
+	}
+
+	// 2. create a self-signed CA certificate.
+	template := &x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName: "registry-addon",
+		},
+		NotBefore:             time.Now().Add(-1 * defaultCertificateNotBeforeSkew),
+		NotAfter:              time.Now().Add(defaultRootCADuration),
+		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+	}
+
+	certDER, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create certificate: %w", err)
+	}
+
+	certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
+	keyPEM = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)})
+
+	return certPEM, keyPEM, nil
+}
+
+func buildRegistryAddonRootCASecret(
+	certPEM,
+	keyPEM []byte,
+	managementCluster *clusterv1.Cluster,
+) *corev1.Secret {
+	data := map[string][]byte{
+		caCrtKey:                certPEM,
+		corev1.TLSCertKey:       certPEM,
+		corev1.TLSPrivateKeyKey: keyPEM,
+	}
+	return &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: corev1.SchemeGroupVersion.String(),
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      handlersutils.RegistryAddonRootCASecretName,
+			Namespace: managementCluster.Namespace,
+		},
+		Data: data,
+	}
+}
+
 type EnsureCertificateOpts struct {
 	// RemoteSecretKey is the name and namespace of the TLS secret to be created on the remote cluster.
 	RemoteSecretKey ctrlclient.ObjectKey

pkg/handlers/generic/lifecycle/registry/utils/tls_integration_test.go

@@ -18,6 +18,7 @@ import (
 	"sigs.k8s.io/cluster-api/controllers/remote"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	handlersutils "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/utils"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
 )
 
@@ -73,6 +74,98 @@ FYnq6/jDVxCbWmmP2u4TT557gMqao0DaJstf/NSXlK0bhA2B64M=
 	testRegistryAddonRootCASecretName = "registry-addon-root-ca"
 )
 
+var _ = Describe("Test EnsureRegistryAddonRootCASecret", func() {
+	clientScheme := runtime.NewScheme()
+	utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
+	utilruntime.Must(clusterv1.AddToScheme(clientScheme))
+
+	It("new root CA Secret should be created", func(ctx SpecContext) {
+		c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
+		Expect(err).To(BeNil())
+
+		cluster := &clusterv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "test-management-cluster-",
+				Namespace:    corev1.NamespaceDefault,
+			},
+		}
+		Expect(c.Create(ctx, cluster)).To(Succeed())
+
+		err = EnsureRegistryAddonRootCASecret(ctx, c, cluster)
+		Expect(err).To(Succeed())
+
+		// Verify the global TLS certificate secret is created.
+		globalTLSCertificateSecret, err := handlersutils.SecretForRegistryAddonRootCA(ctx, c)
+		Expect(err).To(Succeed())
+		Expect(globalTLSCertificateSecret.OwnerReferences).To(
+			ContainElement(
+				metav1.OwnerReference{
+					APIVersion: clusterv1.GroupVersion.String(),
+					Kind:       clusterv1.ClusterKind,
+					Name:       cluster.Name,
+					UID:        cluster.UID,
+				},
+			),
+		)
+		Expect(globalTLSCertificateSecret.Data["ca.crt"]).ToNot(BeEmpty())
+		Expect(globalTLSCertificateSecret.Data[corev1.TLSCertKey]).ToNot(BeEmpty())
+		Expect(globalTLSCertificateSecret.Data[corev1.TLSPrivateKeyKey]).ToNot(BeEmpty())
+	})
+
+	It("a root CA Secret should not be re-created", func(ctx SpecContext) {
+		c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
+		Expect(err).To(BeNil())
+
+		cluster := &clusterv1.Cluster{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "test-management-cluster-",
+				Namespace:    corev1.NamespaceDefault,
+			},
+		}
+		Expect(c.Create(ctx, cluster)).To(Succeed())
+
+		err = EnsureRegistryAddonRootCASecret(ctx, c, cluster)
+		Expect(err).To(Succeed())
+
+		globalTLSCertificateSecret, err := handlersutils.SecretForRegistryAddonRootCA(ctx, c)
+		Expect(err).To(Succeed())
+		Expect(globalTLSCertificateSecret.Data["ca.crt"]).ToNot(BeEmpty())
+		Expect(globalTLSCertificateSecret.Data[corev1.TLSCertKey]).ToNot(BeEmpty())
+		Expect(globalTLSCertificateSecret.Data[corev1.TLSPrivateKeyKey]).ToNot(BeEmpty())
+		data := globalTLSCertificateSecret.Data
+
+		// Verify the data is not changed when running the function again.
+		err = EnsureRegistryAddonRootCASecret(ctx, c, cluster)
+		Expect(err).To(Succeed())
+
+		globalTLSCertificateSecret, err = handlersutils.SecretForRegistryAddonRootCA(ctx, c)
+		Expect(err).To(Succeed())
+
+		Expect(globalTLSCertificateSecret.Data).To(Equal(data))
+	})
+	AfterEach(func(ctx SpecContext) {
+		c, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
+		Expect(err).To(BeNil())
+
+		globalSecret := &corev1.Secret{
+			TypeMeta: metav1.TypeMeta{
+				APIVersion: corev1.SchemeGroupVersion.String(),
+				Kind:       "Secret",
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      handlersutils.RegistryAddonRootCASecretName,
+				Namespace: corev1.NamespaceDefault,
+			},
+		}
+		Expect(c.Delete(ctx, globalSecret)).To(
+			Or(
+				Succeed(),
+				MatchError("secrets \"registry-addon-root-ca\" not found"),
+			),
+		)
+	})
+})
+
 var _ = Describe("Test EnsureCASecretForCluster", func() {
 	clientScheme := runtime.NewScheme()
 	utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))

pkg/handlers/utils/secrets.go

@@ -5,6 +5,7 @@ package utils
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
@@ -19,7 +20,7 @@ import (
 )
 
 const (
-	registryAddonRootCASecretName = "registry-addon-root-ca"
+	RegistryAddonRootCASecretName = "registry-addon-root-ca"
 )
 
 // CopySecretToRemoteCluster will get the Secret from srcSecretName
@@ -157,7 +158,7 @@ func SecretForRegistryAddonRootCA(
 	ctx context.Context,
 	c ctrlclient.Reader,
 ) (*corev1.Secret, error) {
-	secret, err := getSecret(ctx, c, registryAddonRootCASecretName, GetDeploymentNamespace())
+	secret, err := findSecret(ctx, c, RegistryAddonRootCASecretName)
 	if err != nil {
 		return nil, fmt.Errorf("error getting registry addon root CA secret: %w", err)
 	}
@@ -208,3 +209,54 @@ func getSecret(
 	}
 	return secret, c.Get(ctx, ctrlclient.ObjectKeyFromObject(secret), secret)
 }
+
+type ErrSecretNotFound struct {
+	name string
+}
+
+func (e *ErrSecretNotFound) Error() string {
+	return fmt.Sprintf("secrets %q not found", e.name)
+}
+
+func IsSecretNotFoundError(err error) bool {
+	var e *ErrSecretNotFound
+	return errors.As(err, &e)
+}
+
+type ErrMultipleSecretsFound struct {
+	name string
+}
+
+func (e *ErrMultipleSecretsFound) Error() string {
+	return fmt.Sprintf("multiple Secrets found with name %q", e.name)
+}
+
+// findSecret finds a Secret by name across all namespaces.
+// It returns an error if multiple Secrets with the same name are found.
+// Returns nil if no Secret is found.
+func findSecret(
+	ctx context.Context,
+	c ctrlclient.Reader,
+	secretName string,
+) (*corev1.Secret, error) {
+	secrets := &corev1.SecretList{}
+	err := c.List(ctx, secrets)
+	if err != nil {
+		return nil, fmt.Errorf("error listing secrets: %w", err)
+	}
+
+	matches := make([]corev1.Secret, 0)
+	for _, secret := range secrets.Items {
+		if secret.Name == secretName {
+			matches = append(matches, secret)
+		}
+	}
+	switch len(matches) {
+	case 1:
+		return &matches[0], nil
+	case 0:
+		return nil, &ErrSecretNotFound{name: secretName}
+	default:
+		return nil, &ErrMultipleSecretsFound{name: secretName}
+	}
+}