🌱 Test BYO certificates (#10681)

lentzi90 · adilGhaffarDev · web-flow · commit d7bb304240f0 · 2025-01-23T05:25:01.000-08:00
* Test BYO certificates

This adds an integration test for BYO CA certificate.
It is practically a copy of TestReconcileInitializeControlPlane
but with a custom CA and a check to verify that the generated kubeconfig
is really using this CA.

Signed-off-by: Lennart Jern &lt;lennart.jern@est.tech&gt;

* Addressing comments

Signed-off-by: Muhammad Adil Ghaffar &lt;muhammad.adil.ghaffar@est.tech&gt;

* Use individual cache options for test suites

Signed-off-by: Lennart Jern &lt;lennart.jern@est.tech&gt;

---------

Signed-off-by: Lennart Jern &lt;lennart.jern@est.tech&gt;
Signed-off-by: Muhammad Adil Ghaffar &lt;muhammad.adil.ghaffar@est.tech&gt;
Co-authored-by: Muhammad Adil Ghaffar &lt;muhammad.adil.ghaffar@est.tech&gt;
diff --git a/controlplane/kubeadm/internal/controllers/controller_test.go b/controlplane/kubeadm/internal/controllers/controller_test.go
@@ -24,6 +24,7 @@ import (
 	"crypto/x509/pkix"
 	"fmt"
 	"math/big"
+	"path"
 	"sync"
 	"testing"
 	"time"
@@ -37,6 +38,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -1406,6 +1408,246 @@ kubernetesVersion: metav1.16.1
 	}, 30*time.Second).Should(Succeed())
 }
 
+func TestReconcileInitializeControlPlane_withUserCA(t *testing.T) {
+	setup := func(t *testing.T, g *WithT) *corev1.Namespace {
+		t.Helper()
+
+		t.Log("Creating the namespace")
+		ns, err := env.CreateNamespace(ctx, "test-kcp-reconcile-initializecontrolplane")
+		g.Expect(err).ToNot(HaveOccurred())
+
+		return ns
+	}
+
+	teardown := func(t *testing.T, g *WithT, ns *corev1.Namespace) {
+		t.Helper()
+
+		t.Log("Deleting the namespace")
+		g.Expect(env.Delete(ctx, ns)).To(Succeed())
+	}
+
+	g := NewWithT(t)
+	namespace := setup(t, g)
+	defer teardown(t, g, namespace)
+
+	cluster := newCluster(&types.NamespacedName{Name: "foo", Namespace: namespace.Name})
+	cluster.Spec = clusterv1.ClusterSpec{
+		ControlPlaneEndpoint: clusterv1.APIEndpoint{
+			Host: "test.local",
+			Port: 9999,
+		},
+	}
+
+	caCertificate := &secret.Certificate{
+		Purpose:  secret.ClusterCA,
+		CertFile: path.Join(secret.DefaultCertificatesDir, "ca.crt"),
+		KeyFile:  path.Join(secret.DefaultCertificatesDir, "ca.key"),
+	}
+	// The certificate is user provided so no owner references should be added.
+	g.Expect(caCertificate.Generate()).To(Succeed())
+	certSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace.Name,
+			Name:      cluster.Name + "-ca",
+			Labels: map[string]string{
+				clusterv1.ClusterNameLabel: cluster.Name,
+			},
+		},
+		Data: map[string][]byte{
+			secret.TLSKeyDataName: caCertificate.KeyPair.Key,
+			secret.TLSCrtDataName: caCertificate.KeyPair.Cert,
+		},
+		Type: clusterv1.ClusterSecretType,
+	}
+
+	g.Expect(env.Create(ctx, cluster)).To(Succeed())
+	patchHelper, err := patch.NewHelper(cluster, env)
+	g.Expect(err).ToNot(HaveOccurred())
+	cluster.Status = clusterv1.ClusterStatus{InfrastructureReady: true}
+	g.Expect(patchHelper.Patch(ctx, cluster)).To(Succeed())
+
+	g.Expect(env.CreateAndWait(ctx, certSecret)).To(Succeed())
+
+	genericInfrastructureMachineTemplate := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"kind":       "GenericInfrastructureMachineTemplate",
+			"apiVersion": "infrastructure.cluster.x-k8s.io/v1beta1",
+			"metadata": map[string]interface{}{
+				"name":      "infra-foo",
+				"namespace": cluster.Namespace,
+			},
+			"spec": map[string]interface{}{
+				"template": map[string]interface{}{
+					"spec": map[string]interface{}{
+						"hello": "world",
+					},
+				},
+			},
+		},
+	}
+	g.Expect(env.CreateAndWait(ctx, genericInfrastructureMachineTemplate)).To(Succeed())
+
+	kcp := &controlplanev1.KubeadmControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: cluster.Namespace,
+			Name:      "foo",
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					Kind:       "Cluster",
+					APIVersion: clusterv1.GroupVersion.String(),
+					Name:       cluster.Name,
+					UID:        cluster.UID,
+				},
+			},
+		},
+		Spec: controlplanev1.KubeadmControlPlaneSpec{
+			Replicas: nil,
+			Version:  "v1.16.6",
+			MachineTemplate: controlplanev1.KubeadmControlPlaneMachineTemplate{
+				InfrastructureRef: corev1.ObjectReference{
+					Kind:       genericInfrastructureMachineTemplate.GetKind(),
+					APIVersion: genericInfrastructureMachineTemplate.GetAPIVersion(),
+					Name:       genericInfrastructureMachineTemplate.GetName(),
+					Namespace:  cluster.Namespace,
+				},
+			},
+			KubeadmConfigSpec: bootstrapv1.KubeadmConfigSpec{},
+		},
+	}
+	g.Expect(env.Create(ctx, kcp)).To(Succeed())
+
+	corednsCM := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "coredns",
+			Namespace: namespace.Name,
+		},
+		Data: map[string]string{
+			"Corefile": "original-core-file",
+		},
+	}
+	g.Expect(env.Create(ctx, corednsCM)).To(Succeed())
+
+	kubeadmCM := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kubeadm-config",
+			Namespace: namespace.Name,
+		},
+		Data: map[string]string{
+			"ClusterConfiguration": `apiServer:
+dns:
+  type: CoreDNS
+imageRepository: registry.k8s.io
+kind: ClusterConfiguration
+kubernetesVersion: metav1.16.1`,
+		},
+	}
+	g.Expect(env.Create(ctx, kubeadmCM)).To(Succeed())
+
+	corednsDepl := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "coredns",
+			Namespace: namespace.Name,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{
+					"coredns": "",
+				},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "coredns",
+					Labels: map[string]string{
+						"coredns": "",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{
+						Name:  "coredns",
+						Image: "registry.k8s.io/coredns:1.6.2",
+					}},
+				},
+			},
+		},
+	}
+	g.Expect(env.Create(ctx, corednsDepl)).To(Succeed())
+
+	r := &KubeadmControlPlaneReconciler{
+		Client:              env,
+		SecretCachingClient: secretCachingClient,
+		recorder:            record.NewFakeRecorder(32),
+		managementCluster: &fakeManagementCluster{
+			Management: &internal.Management{Client: env},
+			Workload: &fakeWorkloadCluster{
+				Workload: &internal.Workload{
+					Client: env,
+				},
+				Status: internal.ClusterStatus{},
+			},
+		},
+		managementClusterUncached: &fakeManagementCluster{
+			Management: &internal.Management{Client: env},
+			Workload: &fakeWorkloadCluster{
+				Workload: &internal.Workload{
+					Client: env,
+				},
+				Status: internal.ClusterStatus{},
+			},
+		},
+		ssaCache: ssa.NewCache("test-controller"),
+	}
+
+	result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: util.ObjectKey(kcp)})
+	g.Expect(err).ToNot(HaveOccurred())
+	// this first requeue is to add finalizer
+	g.Expect(result).To(BeComparableTo(ctrl.Result{}))
+	g.Expect(env.GetAPIReader().Get(ctx, util.ObjectKey(kcp), kcp)).To(Succeed())
+	g.Expect(kcp.Finalizers).To(ContainElement(controlplanev1.KubeadmControlPlaneFinalizer))
+
+	g.Eventually(func(g Gomega) {
+		_, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: util.ObjectKey(kcp)})
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(env.GetAPIReader().Get(ctx, client.ObjectKey{Name: kcp.Name, Namespace: kcp.Namespace}, kcp)).To(Succeed())
+		// Expect the referenced infrastructure template to have a Cluster Owner Reference.
+		g.Expect(env.GetAPIReader().Get(ctx, util.ObjectKey(genericInfrastructureMachineTemplate), genericInfrastructureMachineTemplate)).To(Succeed())
+		g.Expect(genericInfrastructureMachineTemplate.GetOwnerReferences()).To(ContainElement(metav1.OwnerReference{
+			APIVersion: clusterv1.GroupVersion.String(),
+			Kind:       "Cluster",
+			Name:       cluster.Name,
+			UID:        cluster.UID,
+		}))
+
+		// Always expect that the Finalizer is set on the passed in resource
+		g.Expect(kcp.Finalizers).To(ContainElement(controlplanev1.KubeadmControlPlaneFinalizer))
+
+		g.Expect(kcp.Status.Selector).NotTo(BeEmpty())
+		g.Expect(kcp.Status.Replicas).To(BeEquivalentTo(1))
+		g.Expect(conditions.IsFalse(kcp, controlplanev1.AvailableCondition)).To(BeTrue())
+
+		// Verify that the kubeconfig is using the custom CA
+		kBytes, err := kubeconfig.FromSecret(ctx, env, util.ObjectKey(cluster))
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(kBytes).NotTo(BeEmpty())
+		k, err := clientcmd.Load(kBytes)
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(k).NotTo(BeNil())
+		g.Expect(k.Clusters[cluster.Name]).NotTo(BeNil())
+		g.Expect(k.Clusters[cluster.Name].CertificateAuthorityData).To(Equal(caCertificate.KeyPair.Cert))
+
+		machineList := &clusterv1.MachineList{}
+		g.Expect(env.GetAPIReader().List(ctx, machineList, client.InNamespace(cluster.Namespace))).To(Succeed())
+		g.Expect(machineList.Items).To(HaveLen(1))
+
+		machine := machineList.Items[0]
+		g.Expect(machine.Name).To(HavePrefix(kcp.Name))
+		// Newly cloned infra objects should have the infraref annotation.
+		infraObj, err := external.Get(ctx, r.Client, &machine.Spec.InfrastructureRef)
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(infraObj.GetAnnotations()).To(HaveKeyWithValue(clusterv1.TemplateClonedFromNameAnnotation, genericInfrastructureMachineTemplate.GetName()))
+		g.Expect(infraObj.GetAnnotations()).To(HaveKeyWithValue(clusterv1.TemplateClonedFromGroupKindAnnotation, genericInfrastructureMachineTemplate.GroupVersionKind().GroupKind().String()))
+	}, 30*time.Second).Should(Succeed())
+}
+
 func TestKubeadmControlPlaneReconciler_syncMachines(t *testing.T) {
 	setup := func(t *testing.T, g *WithT) (*corev1.Namespace, *clusterv1.Cluster) {
 		t.Helper()
diff --git a/controlplane/kubeadm/internal/controllers/suite_test.go b/controlplane/kubeadm/internal/controllers/suite_test.go
@@ -23,9 +23,13 @@ import (
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/internal/test/envtest"
 )
 
@@ -48,8 +52,19 @@ func TestMain(m *testing.M) {
 			panic(fmt.Sprintf("unable to create secretCachingClient: %v", err))
 		}
 	}
+	req, _ := labels.NewRequirement(clusterv1.ClusterNameLabel, selection.Exists, nil)
+	clusterSecretCacheSelector := labels.NewSelector().Add(*req)
 	os.Exit(envtest.Run(ctx, envtest.RunInput{
 		M: m,
+		ManagerCacheOptions: cache.Options{
+			ByObject: map[client.Object]cache.ByObject{
+				// Only cache Secrets with the cluster name label.
+				// This is similar to the real world.
+				&corev1.Secret{}: {
+					Label: clusterSecretCacheSelector,
+				},
+			},
+		},
 		ManagerUncachedObjs: []client.Object{
 			&corev1.ConfigMap{},
 			&corev1.Secret{},
diff --git a/exp/addons/internal/controllers/suite_test.go b/exp/addons/internal/controllers/suite_test.go
@@ -25,12 +25,15 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/api/v1beta1/index"
 	"sigs.k8s.io/cluster-api/controllers/clustercache"
 	"sigs.k8s.io/cluster-api/controllers/remote"
@@ -113,9 +116,20 @@ func TestMain(m *testing.M) {
 		}
 	}
 
+	req, _ := labels.NewRequirement(clusterv1.ClusterNameLabel, selection.Exists, nil)
+	clusterSecretCacheSelector := labels.NewSelector().Add(*req)
 	os.Exit(envtest.Run(ctx, envtest.RunInput{
 		M:        m,
 		SetupEnv: func(e *envtest.Environment) { env = e },
+		ManagerCacheOptions: cache.Options{
+			ByObject: map[client.Object]cache.ByObject{
+				// Only cache Secrets with the cluster name label.
+				// This is similar to the real world.
+				&corev1.Secret{}: {
+					Label: clusterSecretCacheSelector,
+				},
+			},
+		},
 		ManagerUncachedObjs: []client.Object{
 			&corev1.ConfigMap{},
 			&corev1.Secret{},
diff --git a/internal/test/envtest/environment.go b/internal/test/envtest/environment.go
@@ -46,6 +46,7 @@ import (
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
@@ -122,6 +123,7 @@ func init() {
 type RunInput struct {
 	M                   *testing.M
 	ManagerUncachedObjs []client.Object
+	ManagerCacheOptions cache.Options
 	SetupIndexes        func(ctx context.Context, mgr ctrl.Manager)
 	SetupReconcilers    func(ctx context.Context, mgr ctrl.Manager)
 	SetupEnv            func(e *Environment)
@@ -147,7 +149,7 @@ func Run(ctx context.Context, input RunInput) int {
 	}
 
 	// Bootstrapping test environment
-	env := newEnvironment(input.ManagerUncachedObjs...)
+	env := newEnvironment(input.ManagerCacheOptions, input.ManagerUncachedObjs...)
 
 	if input.SetupIndexes != nil {
 		input.SetupIndexes(ctx, env.Manager)
@@ -229,7 +231,7 @@ type Environment struct {
 //
 // This function should be called only once for each package you're running tests within,
 // usually the environment is initialized in a suite_test.go file within a `BeforeSuite` ginkgo block.
-func newEnvironment(uncachedObjs ...client.Object) *Environment {
+func newEnvironment(managerCacheOptions cache.Options, uncachedObjs ...client.Object) *Environment {
 	// Get the root of the current file to use in CRD paths.
 	_, filename, _, _ := goruntime.Caller(0) //nolint:dogsled
 	root := path.Join(path.Dir(filename), "..", "..", "..")
@@ -310,6 +312,7 @@ func newEnvironment(uncachedObjs ...client.Object) *Environment {
 				Host:    host,
 			},
 		),
+		Cache: managerCacheOptions,
 	}
 
 	mgr, err := ctrl.NewManager(env.Config, options)
