Skip to content

Commit ccb1020

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #4195 from cnmcavoy/cnmcavoy/eksconfig-reconciliation
Reconcile EKSConfig correctly for MachinePool and other Owner kinds
2 parents fd5e11f + 589b726 commit ccb1020

File tree

3 files changed

+137
-21
lines changed

3 files changed

+137
-21
lines changed

‎bootstrap/eks/controllers/eksconfig_controller.go

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package controllers
1919
import (
2020
"bytes"
2121
"context"
22+
"time"
2223

2324
"github.com/pkg/errors"
2425
corev1 "k8s.io/api/core/v1"
@@ -74,33 +75,36 @@ func (r *EKSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
7475
log.Error(err, "Failed to get config")
7576
return ctrl.Result{}, err
7677
}
78+
log = log.WithValues("EKSConfig", config.GetName())
7779

7880
// check owner references and look up owning Machine object
7981
configOwner, err := bsutil.GetConfigOwner(ctx, r.Client, config)
8082
if apierrors.IsNotFound(err) {
8183
// no error here, requeue until we find an owner
82-
return ctrl.Result{}, nil
84+
log.Debug("eksconfig failed to look up owner reference, re-queueing")
85+
return ctrl.Result{RequeueAfter: time.Minute}, nil
8386
}
8487
if err != nil {
85-
log.Error(err, "Failed to get owner")
88+
log.Error(err, "eksconfig failed to get owner")
8689
return ctrl.Result{}, err
8790
}
8891
if configOwner == nil {
8992
// no error, requeue until we find an owner
90-
return ctrl.Result{}, nil
93+
log.Debug("eksconfig has no owner reference set, re-queueing")
94+
return ctrl.Result{RequeueAfter: time.Minute}, nil
9195
}
9296

9397
log = log.WithValues(configOwner.GetKind(), configOwner.GetName())
9498

9599
cluster, err := util.GetClusterByName(ctx, r.Client, configOwner.GetNamespace(), configOwner.ClusterName())
96100
if err != nil {
97101
if errors.Is(err, util.ErrNoCluster) {
98-
log.Info("EKSConfig does not belong to a cluster yet, re-queuing until it's partof a cluster")
99-
return ctrl.Result{}, nil
102+
log.Info("EKSConfig does not belong to a cluster yet, re-queuing until it's part of a cluster")
103+
return ctrl.Result{RequeueAfter: time.Minute}, nil
100104
}
101105
if apierrors.IsNotFound(err) {
102106
log.Info("Cluster does not exist yet, re-queueing until it is created")
103-
return ctrl.Result{}, nil
107+
return ctrl.Result{RequeueAfter: time.Minute}, nil
104108
}
105109
log.Error(err, "Could not get cluster with metadata")
106110
return ctrl.Result{}, err
@@ -138,13 +142,14 @@ func (r *EKSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
138142
}
139143
}()
140144

141-
return r.joinWorker(ctx, cluster, config)
145+
return r.joinWorker(ctx, cluster, config, configOwner)
142146
}
143147

144-
func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1.Cluster, config *eksbootstrapv1.EKSConfig) (ctrl.Result, error) {
148+
func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1.Cluster, config *eksbootstrapv1.EKSConfig, configOwner *bsutil.ConfigOwner) (ctrl.Result, error) {
145149
log := logger.FromContext(ctx)
146150

147-
if config.Status.DataSecretName != nil {
151+
// only need to reconcile the secret for Machine kinds once, but MachinePools need updates for new launch templates
152+
if config.Status.DataSecretName != nil && configOwner.GetKind() == "Machine" {
148153
secretKey := client.ObjectKey{Namespace: config.Namespace, Name: *config.Status.DataSecretName}
149154
log = log.WithValues("data-secret-name", secretKey.Name)
150155
existingSecret := &corev1.Secret{}
@@ -289,7 +294,7 @@ func (r *EKSConfigReconciler) storeBootstrapData(ctx context.Context, cluster *c
289294
Namespace: config.Namespace,
290295
}, secret); err != nil {
291296
if apierrors.IsNotFound(err) {
292-
if err := r.createBootstrapSecret(ctx, cluster, config, data); err != nil {
297+
if secret, err = r.createBootstrapSecret(ctx, cluster, config, data); err != nil {
293298
return errors.Wrap(err, "failed to create bootstrap data secret for EKSConfig")
294299
}
295300
log.Info("created bootstrap data secret for EKSConfig", "secret", klog.KObj(secret))
@@ -382,7 +387,7 @@ func (r *EKSConfigReconciler) ClusterToEKSConfigs(o client.Object) []ctrl.Reques
382387
}
383388

384389
// Create the Secret containing bootstrap userdata.
385-
func (r *EKSConfigReconciler) createBootstrapSecret(ctx context.Context, cluster *clusterv1.Cluster, config *eksbootstrapv1.EKSConfig, data []byte) error {
390+
func (r *EKSConfigReconciler) createBootstrapSecret(ctx context.Context, cluster *clusterv1.Cluster, config *eksbootstrapv1.EKSConfig, data []byte) (*corev1.Secret, error) {
386391
secret := &corev1.Secret{
387392
ObjectMeta: metav1.ObjectMeta{
388393
Name: config.Name,
@@ -405,11 +410,14 @@ func (r *EKSConfigReconciler) createBootstrapSecret(ctx context.Context, cluster
405410
},
406411
Type: clusterv1.ClusterSecretType,
407412
}
408-
return r.Client.Create(ctx, secret)
413+
return secret, r.Client.Create(ctx, secret)
409414
}
410415

411416
// Update the userdata in the bootstrap Secret.
412417
func (r *EKSConfigReconciler) updateBootstrapSecret(ctx context.Context, secret *corev1.Secret, data []byte) (bool, error) {
418+
if secret.Data == nil {
419+
secret.Data = make(map[string][]byte)
420+
}
413421
if !bytes.Equal(secret.Data["value"], data) {
414422
secret.Data["value"] = data
415423
return true, r.Client.Update(ctx, secret)

‎bootstrap/eks/controllers/eksconfig_controller_reconciler_test.go

Lines changed: 105 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/internal/userdata"
3232
ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
3333
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
34+
"sigs.k8s.io/cluster-api/exp/api/v1beta1"
3435
"sigs.k8s.io/cluster-api/util"
3536
"sigs.k8s.io/cluster-api/util/conditions"
3637
)
@@ -55,7 +56,7 @@ func TestEKSConfigReconciler(t *testing.T) {
5556
}
5657
t.Logf(fmt.Sprintf("Calling reconcile on cluster '%s' and config '%s' should requeue", cluster.Name, config.Name))
5758
g.Eventually(func(gomega Gomega) {
58-
result, err := reconciler.joinWorker(ctx, cluster, config)
59+
result, err := reconciler.joinWorker(ctx, cluster, config, configOwner("Machine"))
5960
gomega.Expect(err).NotTo(HaveOccurred())
6061
gomega.Expect(result.Requeue).To(BeFalse())
6162
}).Should(Succeed())
@@ -74,16 +75,26 @@ func TestEKSConfigReconciler(t *testing.T) {
7475

7576
g.Expect(string(secret.Data["value"])).To(Equal(string(expectedUserData)))
7677
})
77-
7878
t.Run("Should reconcile an EKSConfig and update data Secret", func(t *testing.T) {
7979
g := NewWithT(t)
8080
amcp := newAMCP("test-cluster")
8181
cluster := newCluster(amcp.Name)
82-
machine := newMachine(cluster, "test-machine")
83-
config := newEKSConfig(machine)
82+
mp := newMachinePool(cluster, "test-machine")
83+
config := newEKSConfig(nil)
84+
config.ObjectMeta.Name = mp.Name
85+
config.ObjectMeta.UID = types.UID(fmt.Sprintf("%s uid", mp.Name))
86+
config.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
87+
{
88+
Kind: "MachinePool",
89+
APIVersion: v1beta1.GroupVersion.String(),
90+
Name: mp.Name,
91+
UID: types.UID(fmt.Sprintf("%s uid", mp.Name)),
92+
},
93+
}
94+
config.Status.DataSecretName = &mp.Name
8495
t.Logf(dump("amcp", amcp))
8596
t.Logf(dump("config", config))
86-
t.Logf(dump("machine", machine))
97+
t.Logf(dump("machinepool", mp))
8798
t.Logf(dump("cluster", cluster))
8899
oldUserData, err := newUserData(cluster.Name, map[string]string{"test-arg": "test-value"})
89100
g.Expect(err).To(BeNil())
@@ -100,7 +111,7 @@ func TestEKSConfigReconciler(t *testing.T) {
100111
}
101112
t.Logf(fmt.Sprintf("Calling reconcile on cluster '%s' and config '%s' should requeue", cluster.Name, config.Name))
102113
g.Eventually(func(gomega Gomega) {
103-
result, err := reconciler.joinWorker(ctx, cluster, config)
114+
result, err := reconciler.joinWorker(ctx, cluster, config, configOwner("MachinePool"))
104115
gomega.Expect(err).NotTo(HaveOccurred())
105116
gomega.Expect(result.Requeue).To(BeFalse())
106117
}).Should(Succeed())
@@ -125,7 +136,7 @@ func TestEKSConfigReconciler(t *testing.T) {
125136
}
126137
t.Logf(dump("config", config))
127138
g.Eventually(func(gomega Gomega) {
128-
result, err := reconciler.joinWorker(ctx, cluster, config)
139+
result, err := reconciler.joinWorker(ctx, cluster, config, configOwner("MachinePool"))
129140
gomega.Expect(err).NotTo(HaveOccurred())
130141
gomega.Expect(result.Requeue).To(BeFalse())
131142
}).Should(Succeed())
@@ -141,6 +152,57 @@ func TestEKSConfigReconciler(t *testing.T) {
141152
gomega.Expect(string(secret.Data["value"])).To(Equal(string(expectedUserData)))
142153
}).Should(Succeed())
143154
})
155+
156+
t.Run("Should reconcile an EKSConfig and not update data if secret exists and config owner is Machine kind", func(t *testing.T) {
157+
g := NewWithT(t)
158+
amcp := newAMCP("test-cluster")
159+
cluster := newCluster(amcp.Name)
160+
machine := newMachine(cluster, "test-machine")
161+
config := newEKSConfig(machine)
162+
t.Logf(dump("amcp", amcp))
163+
t.Logf(dump("config", config))
164+
t.Logf(dump("machine", machine))
165+
t.Logf(dump("cluster", cluster))
166+
expectedUserData, err := newUserData(cluster.Name, map[string]string{"test-arg": "test-value"})
167+
g.Expect(err).To(BeNil())
168+
g.Expect(testEnv.Client.Create(ctx, amcp)).To(Succeed())
169+
170+
secret := &corev1.Secret{
171+
ObjectMeta: metav1.ObjectMeta{
172+
Namespace: "default",
173+
Name: machine.Name,
174+
},
175+
}
176+
g.Expect(testEnv.Client.Create(ctx, secret)).To(Succeed())
177+
178+
amcpList := &ekscontrolplanev1.AWSManagedControlPlaneList{}
179+
testEnv.Client.List(ctx, amcpList)
180+
t.Logf(dump("stored-amcps", amcpList))
181+
182+
reconciler := EKSConfigReconciler{
183+
Client: testEnv.Client,
184+
}
185+
t.Logf(fmt.Sprintf("Calling reconcile on cluster '%s' and config '%s' should requeue", cluster.Name, config.Name))
186+
g.Eventually(func(gomega Gomega) {
187+
result, err := reconciler.joinWorker(ctx, cluster, config, configOwner("Machine"))
188+
gomega.Expect(err).NotTo(HaveOccurred())
189+
gomega.Expect(result.Requeue).To(BeFalse())
190+
}).Should(Succeed())
191+
192+
t.Logf(fmt.Sprintf("Secret '%s' should exist and be out of date", config.Name))
193+
secretList := &corev1.SecretList{}
194+
testEnv.Client.List(ctx, secretList)
195+
t.Logf(dump("secrets", secretList))
196+
197+
secret = &corev1.Secret{}
198+
g.Eventually(func(gomega Gomega) {
199+
gomega.Expect(testEnv.Client.Get(ctx, client.ObjectKey{
200+
Name: config.Name,
201+
Namespace: "default",
202+
}, secret)).To(Succeed())
203+
gomega.Expect(string(secret.Data["value"])).To(Not(Equal(string(expectedUserData))))
204+
}).Should(Succeed())
205+
})
144206
}
145207

146208
// newCluster return a CAPI cluster object.
@@ -204,6 +266,40 @@ func newMachine(cluster *clusterv1.Cluster, name string) *clusterv1.Machine {
204266
return machine
205267
}
206268

269+
// newMachinePool returns a CAPI machine object; if cluster is not nil, the MachinePool is linked to the cluster as well.
270+
func newMachinePool(cluster *clusterv1.Cluster, name string) *v1beta1.MachinePool {
271+
generatedName := fmt.Sprintf("%s-%s", name, util.RandomString(5))
272+
mp := &v1beta1.MachinePool{
273+
TypeMeta: metav1.TypeMeta{
274+
Kind: "MachinePool",
275+
APIVersion: v1beta1.GroupVersion.String(),
276+
},
277+
ObjectMeta: metav1.ObjectMeta{
278+
Namespace: "default",
279+
Name: generatedName,
280+
},
281+
Spec: v1beta1.MachinePoolSpec{
282+
Template: clusterv1.MachineTemplateSpec{
283+
Spec: clusterv1.MachineSpec{
284+
Bootstrap: clusterv1.Bootstrap{
285+
ConfigRef: &corev1.ObjectReference{
286+
Kind: "EKSConfig",
287+
APIVersion: eksbootstrapv1.GroupVersion.String(),
288+
},
289+
},
290+
},
291+
},
292+
},
293+
}
294+
if cluster != nil {
295+
mp.Spec.ClusterName = cluster.Name
296+
mp.ObjectMeta.Labels = map[string]string{
297+
clusterv1.ClusterLabelName: cluster.Name,
298+
}
299+
}
300+
return mp
301+
}
302+
207303
// newEKSConfig return an EKSConfig object; if machine is not nil, the EKSConfig is linked to the machine as well.
208304
func newEKSConfig(machine *clusterv1.Machine) *eksbootstrapv1.EKSConfig {
209305
config := &eksbootstrapv1.EKSConfig{
@@ -219,6 +315,7 @@ func newEKSConfig(machine *clusterv1.Machine) *eksbootstrapv1.EKSConfig {
219315
"test-arg": "test-value",
220316
},
221317
},
318+
Status: eksbootstrapv1.EKSConfigStatus{},
222319
}
223320
if machine != nil {
224321
config.ObjectMeta.Name = machine.Name
@@ -231,6 +328,7 @@ func newEKSConfig(machine *clusterv1.Machine) *eksbootstrapv1.EKSConfig {
231328
UID: types.UID(fmt.Sprintf("%s uid", machine.Name)),
232329
},
233330
}
331+
config.Status.DataSecretName = &machine.Name
234332
machine.Spec.Bootstrap.ConfigRef.Name = config.Name
235333
machine.Spec.Bootstrap.ConfigRef.Namespace = config.Namespace
236334
}

‎bootstrap/eks/controllers/eksconfig_controller_test.go

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,11 @@ import (
2121
"testing"
2222

2323
. "github.com/onsi/gomega"
24+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2425
"sigs.k8s.io/controller-runtime/pkg/reconcile"
2526

2627
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
28+
bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
2729
)
2830

2931
func TestEKSConfigReconcilerReturnEarlyIfClusterInfraNotReady(t *testing.T) {
@@ -42,7 +44,7 @@ func TestEKSConfigReconcilerReturnEarlyIfClusterInfraNotReady(t *testing.T) {
4244
}
4345

4446
g.Eventually(func(gomega Gomega) {
45-
result, err := reconciler.joinWorker(context.Background(), cluster, config)
47+
result, err := reconciler.joinWorker(context.Background(), cluster, config, configOwner("Machine"))
4648
gomega.Expect(result).To(Equal(reconcile.Result{}))
4749
gomega.Expect(err).NotTo(HaveOccurred())
4850
}).Should(Succeed())
@@ -64,8 +66,16 @@ func TestEKSConfigReconcilerReturnEarlyIfClusterControlPlaneNotInitialized(t *te
6466
}
6567

6668
g.Eventually(func(gomega Gomega) {
67-
result, err := reconciler.joinWorker(context.Background(), cluster, config)
69+
result, err := reconciler.joinWorker(context.Background(), cluster, config, configOwner("Machine"))
6870
gomega.Expect(result).To(Equal(reconcile.Result{}))
6971
gomega.Expect(err).NotTo(HaveOccurred())
7072
}).Should(Succeed())
7173
}
74+
75+
func configOwner(kind string) *bsutil.ConfigOwner {
76+
unstructuredOwner := unstructured.Unstructured{
77+
Object: map[string]interface{}{"kind": kind},
78+
}
79+
configOwner := bsutil.ConfigOwner{Unstructured: &unstructuredOwner}
80+
return &configOwner
81+
}

0 commit comments

Comments
 (0)