Skip to content

Commit 479ca19

Browse files
fabriziopandinifabriziopandini
committed
Improve KCP to use only one workload cluster for each reconcile
1 parent 3c9e154 commit 479ca19

File tree

14 files changed

+423
-233
lines changed

14 files changed

+423
-233
lines changed

‎controlplane/kubeadm/internal/control_plane.go

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,13 @@ type ControlPlane struct {
5151
// See discussion on https://github.com/kubernetes-sigs/cluster-api/pull/3405
5252
KubeadmConfigs map[string]*bootstrapv1.KubeadmConfig
5353
InfraResources map[string]*unstructured.Unstructured
54+
55+
managementCluster ManagementCluster
56+
workloadCluster WorkloadCluster
5457
}
5558

5659
// NewControlPlane returns an instantiated ControlPlane.
57-
func NewControlPlane(ctx context.Context, client client.Client, cluster *clusterv1.Cluster, kcp *controlplanev1.KubeadmControlPlane, ownedMachines collections.Machines) (*ControlPlane, error) {
60+
func NewControlPlane(ctx context.Context, managementCluster ManagementCluster, client client.Client, cluster *clusterv1.Cluster, kcp *controlplanev1.KubeadmControlPlane, ownedMachines collections.Machines) (*ControlPlane, error) {
5861
infraObjects, err := getInfraResources(ctx, client, ownedMachines)
5962
if err != nil {
6063
return nil, err
@@ -80,6 +83,7 @@ func NewControlPlane(ctx context.Context, client client.Client, cluster *cluster
8083
KubeadmConfigs: kubeadmConfigs,
8184
InfraResources: infraObjects,
8285
reconciliationTime: metav1.Now(),
86+
managementCluster: managementCluster,
8387
}, nil
8488
}
8589

@@ -268,3 +272,28 @@ func (c *ControlPlane) SetPatchHelpers(patchHelpers map[string]*patch.Helper) {
268272
c.machinesPatchHelpers[machineName] = patchHelper
269273
}
270274
}
275+
276+
// GetWorkloadCluster builds a cluster object.
277+
// The cluster comes with an etcd client generator to connect to any etcd pod living on a managed machine.
278+
func (c *ControlPlane) GetWorkloadCluster(ctx context.Context) (WorkloadCluster, error) {
279+
if c.workloadCluster != nil {
280+
return c.workloadCluster, nil
281+
}
282+
283+
workloadCluster, err := c.managementCluster.GetWorkloadCluster(ctx, client.ObjectKeyFromObject(c.Cluster))
284+
if err != nil {
285+
return nil, err
286+
}
287+
c.workloadCluster = workloadCluster
288+
return c.workloadCluster, nil
289+
}
290+
291+
// InjectTestManagementCluster allows to inject a test ManagementCluster during tests.
292+
// NOTE: This approach allows to keep the managementCluster field private, which will
293+
// prevent people from using managementCluster.GetWorkloadCluster because it creates a new
294+
// instance of WorkloadCluster at every call. People instead should use ControlPlane.GetWorkloadCluster
295+
// that creates only a single instance of WorkloadCluster for each reconcile.
296+
func (c *ControlPlane) InjectTestManagementCluster(managementCluster ManagementCluster) {
297+
c.managementCluster = managementCluster
298+
c.workloadCluster = nil
299+
}

‎controlplane/kubeadm/internal/controllers/controller.go

Lines changed: 130 additions & 91 deletions
Large diffs are not rendered by default.

‎controlplane/kubeadm/internal/controllers/controller_test.go

Lines changed: 71 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -520,7 +520,9 @@ func TestKubeadmControlPlaneReconciler_adoption(t *testing.T) {
520520
managementClusterUncached: fmc,
521521
}
522522

523-
g.Expect(r.reconcile(ctx, cluster, kcp)).To(Equal(ctrl.Result{}))
523+
_, adoptableMachineFound, err := r.initControlPlaneScope(ctx, cluster, kcp)
524+
g.Expect(err).ToNot(HaveOccurred())
525+
g.Expect(adoptableMachineFound).To(BeTrue())
524526

525527
machineList := &clusterv1.MachineList{}
526528
g.Expect(fakeClient.List(ctx, machineList, client.InNamespace(cluster.Namespace))).To(Succeed())
@@ -614,7 +616,9 @@ func TestKubeadmControlPlaneReconciler_adoption(t *testing.T) {
614616
managementClusterUncached: fmc,
615617
}
616618

617-
g.Expect(r.reconcile(ctx, cluster, kcp)).To(Equal(ctrl.Result{}))
619+
_, adoptableMachineFound, err := r.initControlPlaneScope(ctx, cluster, kcp)
620+
g.Expect(err).ToNot(HaveOccurred())
621+
g.Expect(adoptableMachineFound).To(BeTrue())
618622

619623
machineList := &clusterv1.MachineList{}
620624
g.Expect(fakeClient.List(ctx, machineList, client.InNamespace(cluster.Namespace))).To(Succeed())
@@ -698,10 +702,9 @@ func TestKubeadmControlPlaneReconciler_adoption(t *testing.T) {
698702
managementClusterUncached: fmc,
699703
}
700704

701-
result, err := r.reconcile(ctx, cluster, kcp)
702-
g.Expect(result).To(Equal(ctrl.Result{}))
703-
g.Expect(err).To(HaveOccurred())
704-
g.Expect(err.Error()).To(ContainSubstring("has just been deleted"))
705+
_, adoptableMachineFound, err := r.initControlPlaneScope(ctx, cluster, kcp)
706+
g.Expect(err).ToNot(HaveOccurred())
707+
g.Expect(adoptableMachineFound).To(BeFalse())
705708

706709
machineList := &clusterv1.MachineList{}
707710
g.Expect(fakeClient.List(ctx, machineList, client.InNamespace(cluster.Namespace))).To(Succeed())
@@ -711,7 +714,7 @@ func TestKubeadmControlPlaneReconciler_adoption(t *testing.T) {
711714
}
712715
})
713716

714-
t.Run("refuses to adopt Machines that are more than one version old", func(t *testing.T) {
717+
t.Run("Do not adopt Machines that are more than one version old", func(t *testing.T) {
715718
g := NewWithT(t)
716719

717720
cluster, kcp, tmpl := createClusterWithControlPlane(metav1.NamespaceDefault)
@@ -753,7 +756,10 @@ func TestKubeadmControlPlaneReconciler_adoption(t *testing.T) {
753756
managementClusterUncached: fmc,
754757
}
755758

756-
g.Expect(r.reconcile(ctx, cluster, kcp)).To(Equal(ctrl.Result{}))
759+
_, adoptableMachineFound, err := r.initControlPlaneScope(ctx, cluster, kcp)
760+
g.Expect(err).ToNot(HaveOccurred())
761+
g.Expect(adoptableMachineFound).To(BeTrue())
762+
757763
// Message: Warning AdoptionFailed Could not adopt Machine test/test0: its version ("v1.15.0") is outside supported +/- one minor version skew from KCP's ("v1.17.0")
758764
g.Expect(recorder.Events).To(Receive(ContainSubstring("minor version")))
759765

@@ -818,7 +824,8 @@ func TestKubeadmControlPlaneReconciler_ensureOwnerReferences(t *testing.T) {
818824

819825
fakeClient := newFakeClient(objs...)
820826

821-
err = ensureCertificatesOwnerRef(ctx, fakeClient, client.ObjectKeyFromObject(cluster), certificates, kcpOwner)
827+
r := KubeadmControlPlaneReconciler{Client: fakeClient}
828+
err = r.ensureCertificatesOwnerRef(ctx, client.ObjectKeyFromObject(cluster), certificates, kcpOwner)
822829
g.Expect(err).To(BeNil())
823830

824831
secrets := &corev1.SecretList{}
@@ -858,7 +865,9 @@ func TestKubeadmControlPlaneReconciler_ensureOwnerReferences(t *testing.T) {
858865
}
859866

860867
fakeClient := newFakeClient(objs...)
861-
err := ensureCertificatesOwnerRef(ctx, fakeClient, client.ObjectKeyFromObject(cluster), certificates, kcpOwner)
868+
869+
r := KubeadmControlPlaneReconciler{Client: fakeClient}
870+
err := r.ensureCertificatesOwnerRef(ctx, client.ObjectKeyFromObject(cluster), certificates, kcpOwner)
862871
g.Expect(err).To(BeNil())
863872

864873
secrets := &corev1.SecretList{}
@@ -901,7 +910,9 @@ func TestKubeadmControlPlaneReconciler_ensureOwnerReferences(t *testing.T) {
901910
}
902911

903912
fakeClient := newFakeClient(objs...)
904-
err := ensureCertificatesOwnerRef(ctx, fakeClient, client.ObjectKeyFromObject(cluster), certificates, kcpOwner)
913+
914+
r := KubeadmControlPlaneReconciler{Client: fakeClient}
915+
err := r.ensureCertificatesOwnerRef(ctx, client.ObjectKeyFromObject(cluster), certificates, kcpOwner)
905916
g.Expect(err).To(BeNil())
906917

907918
secrets := &corev1.SecretList{}
@@ -1088,18 +1099,20 @@ func TestReconcileCertificateExpiries(t *testing.T) {
10881099
machineWithoutNodeRefKubeadmConfig,
10891100
)
10901101

1091-
controlPlane, err := internal.NewControlPlane(ctx, fakeClient, cluster, kcp, ownedMachines)
1092-
g.Expect(err).ToNot(HaveOccurred())
1102+
managementCluster := &fakeManagementCluster{
1103+
Workload: fakeWorkloadCluster{
1104+
APIServerCertificateExpiry: &detectedExpiry,
1105+
},
1106+
}
10931107

10941108
r := &KubeadmControlPlaneReconciler{
1095-
Client: fakeClient,
1096-
managementCluster: &fakeManagementCluster{
1097-
Workload: fakeWorkloadCluster{
1098-
APIServerCertificateExpiry: &detectedExpiry,
1099-
},
1100-
},
1109+
Client: fakeClient,
1110+
managementCluster: managementCluster,
11011111
}
11021112

1113+
controlPlane, err := internal.NewControlPlane(ctx, managementCluster, fakeClient, cluster, kcp, ownedMachines)
1114+
g.Expect(err).ToNot(HaveOccurred())
1115+
11031116
_, err = r.reconcileCertificateExpiries(ctx, controlPlane)
11041117
g.Expect(err).NotTo(HaveOccurred())
11051118

@@ -2014,9 +2027,11 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
20142027
controllerutil.AddFinalizer(kcp, controlplanev1.KubeadmControlPlaneFinalizer)
20152028
initObjs := []client.Object{cluster.DeepCopy(), kcp.DeepCopy()}
20162029

2030+
machines := collections.New()
20172031
for i := 0; i < 3; i++ {
20182032
m, _ := createMachineNodePair(fmt.Sprintf("test-%d", i), cluster, kcp, true)
20192033
initObjs = append(initObjs, m)
2034+
machines.Insert(m)
20202035
}
20212036

20222037
fakeClient := newFakeClient(initObjs...)
@@ -2031,7 +2046,13 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
20312046
recorder: record.NewFakeRecorder(32),
20322047
}
20332048

2034-
result, err := r.reconcileDelete(ctx, cluster, kcp)
2049+
controlPlane := &internal.ControlPlane{
2050+
KCP: kcp,
2051+
Cluster: cluster,
2052+
Machines: machines,
2053+
}
2054+
2055+
result, err := r.reconcileDelete(ctx, controlPlane)
20352056
g.Expect(result).To(Equal(ctrl.Result{RequeueAfter: deleteRequeueAfter}))
20362057
g.Expect(err).ToNot(HaveOccurred())
20372058
g.Expect(kcp.Finalizers).To(ContainElement(controlplanev1.KubeadmControlPlaneFinalizer))
@@ -2040,7 +2061,12 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
20402061
g.Expect(fakeClient.List(ctx, &controlPlaneMachines)).To(Succeed())
20412062
g.Expect(controlPlaneMachines.Items).To(BeEmpty())
20422063

2043-
result, err = r.reconcileDelete(ctx, cluster, kcp)
2064+
controlPlane = &internal.ControlPlane{
2065+
KCP: kcp,
2066+
Cluster: cluster,
2067+
}
2068+
2069+
result, err = r.reconcileDelete(ctx, controlPlane)
20442070
g.Expect(result).To(Equal(ctrl.Result{}))
20452071
g.Expect(err).NotTo(HaveOccurred())
20462072
g.Expect(kcp.Finalizers).To(BeEmpty())
@@ -2064,9 +2090,11 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
20642090

20652091
initObjs := []client.Object{cluster.DeepCopy(), kcp.DeepCopy(), workerMachine.DeepCopy()}
20662092

2093+
machines := collections.New()
20672094
for i := 0; i < 3; i++ {
20682095
m, _ := createMachineNodePair(fmt.Sprintf("test-%d", i), cluster, kcp, true)
20692096
initObjs = append(initObjs, m)
2097+
machines.Insert(m)
20702098
}
20712099

20722100
fakeClient := newFakeClient(initObjs...)
@@ -2080,7 +2108,13 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
20802108
recorder: record.NewFakeRecorder(32),
20812109
}
20822110

2083-
result, err := r.reconcileDelete(ctx, cluster, kcp)
2111+
controlPlane := &internal.ControlPlane{
2112+
KCP: kcp,
2113+
Cluster: cluster,
2114+
Machines: machines,
2115+
}
2116+
2117+
result, err := r.reconcileDelete(ctx, controlPlane)
20842118
g.Expect(result).To(Equal(ctrl.Result{RequeueAfter: deleteRequeueAfter}))
20852119
g.Expect(err).ToNot(HaveOccurred())
20862120

@@ -2113,9 +2147,11 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
21132147

21142148
initObjs := []client.Object{cluster.DeepCopy(), kcp.DeepCopy(), workerMachinePool.DeepCopy()}
21152149

2150+
machines := collections.New()
21162151
for i := 0; i < 3; i++ {
21172152
m, _ := createMachineNodePair(fmt.Sprintf("test-%d", i), cluster, kcp, true)
21182153
initObjs = append(initObjs, m)
2154+
machines.Insert(m)
21192155
}
21202156

21212157
fakeClient := newFakeClient(initObjs...)
@@ -2129,7 +2165,13 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
21292165
recorder: record.NewFakeRecorder(32),
21302166
}
21312167

2132-
result, err := r.reconcileDelete(ctx, cluster, kcp)
2168+
controlPlane := &internal.ControlPlane{
2169+
KCP: kcp,
2170+
Cluster: cluster,
2171+
Machines: machines,
2172+
}
2173+
2174+
result, err := r.reconcileDelete(ctx, controlPlane)
21332175
g.Expect(result).To(Equal(ctrl.Result{RequeueAfter: deleteRequeueAfter}))
21342176
g.Expect(err).ToNot(HaveOccurred())
21352177

@@ -2160,7 +2202,12 @@ func TestKubeadmControlPlaneReconciler_reconcileDelete(t *testing.T) {
21602202
recorder: record.NewFakeRecorder(32),
21612203
}
21622204

2163-
result, err := r.reconcileDelete(ctx, cluster, kcp)
2205+
controlPlane := &internal.ControlPlane{
2206+
KCP: kcp,
2207+
Cluster: cluster,
2208+
}
2209+
2210+
result, err := r.reconcileDelete(ctx, controlPlane)
21642211
g.Expect(result).To(Equal(ctrl.Result{}))
21652212
g.Expect(err).NotTo(HaveOccurred())
21662213
g.Expect(kcp.Finalizers).To(BeEmpty())

‎controlplane/kubeadm/internal/controllers/helpers.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,16 +47,16 @@ import (
4747
"sigs.k8s.io/cluster-api/util/secret"
4848
)
4949

50-
func (r *KubeadmControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, cluster *clusterv1.Cluster, kcp *controlplanev1.KubeadmControlPlane) (ctrl.Result, error) {
50+
func (r *KubeadmControlPlaneReconciler) reconcileKubeconfig(ctx context.Context, controlPlane *internal.ControlPlane) (ctrl.Result, error) {
5151
log := ctrl.LoggerFrom(ctx)
5252

53-
endpoint := cluster.Spec.ControlPlaneEndpoint
53+
endpoint := controlPlane.Cluster.Spec.ControlPlaneEndpoint
5454
if endpoint.IsZero() {
5555
return ctrl.Result{}, nil
5656
}
5757

58-
controllerOwnerRef := *metav1.NewControllerRef(kcp, controlplanev1.GroupVersion.WithKind(kubeadmControlPlaneKind))
59-
clusterName := util.ObjectKey(cluster)
58+
controllerOwnerRef := *metav1.NewControllerRef(controlPlane.KCP, controlplanev1.GroupVersion.WithKind(kubeadmControlPlaneKind))
59+
clusterName := util.ObjectKey(controlPlane.Cluster)
6060
configSecret, err := secret.GetFromNamespacedName(ctx, r.Client, clusterName, secret.Kubeconfig)
6161
switch {
6262
case apierrors.IsNotFound(err):
@@ -76,12 +76,12 @@ func (r *KubeadmControlPlaneReconciler) reconcileKubeconfig(ctx context.Context,
7676
return ctrl.Result{}, errors.Wrap(err, "failed to retrieve kubeconfig Secret")
7777
}
7878

79-
if err := r.adoptKubeconfigSecret(ctx, configSecret, kcp); err != nil {
79+
if err := r.adoptKubeconfigSecret(ctx, configSecret, controlPlane.KCP); err != nil {
8080
return ctrl.Result{}, err
8181
}
8282

8383
// only do rotation on owned secrets
84-
if !util.IsControlledBy(configSecret, kcp) {
84+
if !util.IsControlledBy(configSecret, controlPlane.KCP) {
8585
return ctrl.Result{}, nil
8686
}
8787

‎controlplane/kubeadm/internal/controllers/helpers_test.go

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
3535
"sigs.k8s.io/cluster-api/controllers/external"
3636
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
37+
"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
3738
"sigs.k8s.io/cluster-api/util/conditions"
3839
"sigs.k8s.io/cluster-api/util/kubeconfig"
3940
"sigs.k8s.io/cluster-api/util/secret"
@@ -77,7 +78,12 @@ func TestReconcileKubeconfigEmptyAPIEndpoints(t *testing.T) {
7778
recorder: record.NewFakeRecorder(32),
7879
}
7980

80-
result, err := r.reconcileKubeconfig(ctx, cluster, kcp)
81+
controlPlane := &internal.ControlPlane{
82+
KCP: kcp,
83+
Cluster: cluster,
84+
}
85+
86+
result, err := r.reconcileKubeconfig(ctx, controlPlane)
8187
g.Expect(err).ToNot(HaveOccurred())
8288
g.Expect(result).To(BeZero())
8389

@@ -126,7 +132,12 @@ func TestReconcileKubeconfigMissingCACertificate(t *testing.T) {
126132
recorder: record.NewFakeRecorder(32),
127133
}
128134

129-
result, err := r.reconcileKubeconfig(ctx, cluster, kcp)
135+
controlPlane := &internal.ControlPlane{
136+
KCP: kcp,
137+
Cluster: cluster,
138+
}
139+
140+
result, err := r.reconcileKubeconfig(ctx, controlPlane)
130141
g.Expect(err).ToNot(HaveOccurred())
131142
g.Expect(result).To(Equal(ctrl.Result{RequeueAfter: dependentCertRequeueAfter}))
132143

@@ -192,7 +203,12 @@ func TestReconcileKubeconfigSecretDoesNotAdoptsUserSecrets(t *testing.T) {
192203
recorder: record.NewFakeRecorder(32),
193204
}
194205

195-
result, err := r.reconcileKubeconfig(ctx, cluster, kcp)
206+
controlPlane := &internal.ControlPlane{
207+
KCP: kcp,
208+
Cluster: cluster,
209+
}
210+
211+
result, err := r.reconcileKubeconfig(ctx, controlPlane)
196212
g.Expect(err).To(Succeed())
197213
g.Expect(result).To(BeZero())
198214

@@ -251,7 +267,13 @@ func TestKubeadmControlPlaneReconciler_reconcileKubeconfig(t *testing.T) {
251267
Client: fakeClient,
252268
recorder: record.NewFakeRecorder(32),
253269
}
254-
result, err := r.reconcileKubeconfig(ctx, cluster, kcp)
270+
271+
controlPlane := &internal.ControlPlane{
272+
KCP: kcp,
273+
Cluster: cluster,
274+
}
275+
276+
result, err := r.reconcileKubeconfig(ctx, controlPlane)
255277
g.Expect(err).ToNot(HaveOccurred())
256278
g.Expect(result).To(Equal(ctrl.Result{}))
257279

0 commit comments

Comments
 (0)