Skip to content

Commit 51a4011

Browse files
AndiDogAndiDog
committed
Recreate bootstrap token if it was cleaned up
1 parent 18085c3 commit 51a4011

File tree

2 files changed

+177
-12
lines changed

2 files changed

+177
-12
lines changed

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go‎

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ func (r *KubeadmConfigReconciler) reconcile(ctx context.Context, scope *Scope, c
322322
// If the BootstrapToken has been generated for a join but the config owner has no nodeRefs,
323323
// this indicates that the node has not yet joined and the token in the join config has not
324324
// been consumed and it may need a refresh.
325-
return r.refreshBootstrapTokenIfNeeded(ctx, config, cluster)
325+
return r.refreshBootstrapTokenIfNeeded(ctx, config, cluster, scope)
326326
}
327327
if configOwner.IsMachinePool() {
328328
// If the BootstrapToken has been generated and infrastructure is ready but the configOwner is a MachinePool,
@@ -360,7 +360,7 @@ func (r *KubeadmConfigReconciler) reconcile(ctx context.Context, scope *Scope, c
360360
return r.joinWorker(ctx, scope)
361361
}
362362

363-
func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster) (ctrl.Result, error) {
363+
func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster, scope *Scope) (ctrl.Result, error) {
364364
log := ctrl.LoggerFrom(ctx)
365365
token := config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
366366

@@ -371,6 +371,11 @@ func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Cont
371371

372372
secret, err := getToken(ctx, remoteClient, token)
373373
if err != nil {
374+
if apierrors.IsNotFound(err) && scope.ConfigOwner.IsMachinePool() {
375+
log.Info("Bootstrap token secret not found, triggering creation of new token")
376+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = ""
377+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
378+
}
374379
return ctrl.Result{}, errors.Wrapf(err, "failed to get bootstrap token secret in order to refresh it")
375380
}
376381
log = log.WithValues("Secret", klog.KObj(secret))
@@ -401,13 +406,33 @@ func (r *KubeadmConfigReconciler) refreshBootstrapTokenIfNeeded(ctx context.Cont
401406
log.Info("Refreshing token until the infrastructure has a chance to consume it", "oldExpiration", secretExpiration, "newExpiration", newExpiration)
402407
err = remoteClient.Update(ctx, secret)
403408
if err != nil {
409+
if apierrors.IsNotFound(err) && scope.ConfigOwner.IsMachinePool() {
410+
log.Info("Bootstrap token secret not found, triggering creation of new token")
411+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = ""
412+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
413+
}
404414
return ctrl.Result{}, errors.Wrapf(err, "failed to refresh bootstrap token")
405415
}
406416
return ctrl.Result{
407417
RequeueAfter: r.tokenCheckRefreshOrRotationInterval(),
408418
}, nil
409419
}
410420

421+
func (r *KubeadmConfigReconciler) recreateBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, scope *Scope, remoteClient client.Client) (ctrl.Result, error) {
422+
log := ctrl.LoggerFrom(ctx)
423+
424+
token, err := createToken(ctx, remoteClient, r.TokenTTL)
425+
if err != nil {
426+
return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
427+
}
428+
429+
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
430+
log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
431+
432+
// Update the bootstrap data
433+
return r.joinWorker(ctx, scope)
434+
}
435+
411436
func (r *KubeadmConfigReconciler) rotateMachinePoolBootstrapToken(ctx context.Context, config *bootstrapv1.KubeadmConfig, cluster *clusterv1.Cluster, scope *Scope) (ctrl.Result, error) {
412437
log := ctrl.LoggerFrom(ctx)
413438
log.V(2).Info("Config is owned by a MachinePool, checking if token should be rotated")
@@ -423,16 +448,7 @@ func (r *KubeadmConfigReconciler) rotateMachinePoolBootstrapToken(ctx context.Co
423448
}
424449
if shouldRotate {
425450
log.Info("Creating new bootstrap token, the existing one should be rotated")
426-
token, err := createToken(ctx, remoteClient, r.TokenTTL)
427-
if err != nil {
428-
return ctrl.Result{}, errors.Wrapf(err, "failed to create new bootstrap token")
429-
}
430-
431-
config.Spec.JoinConfiguration.Discovery.BootstrapToken.Token = token
432-
log.V(3).Info("Altering JoinConfiguration.Discovery.BootstrapToken.Token")
433-
434-
// update the bootstrap data
435-
return r.joinWorker(ctx, scope)
451+
return r.recreateBootstrapToken(ctx, config, scope, remoteClient)
436452
}
437453
return ctrl.Result{
438454
RequeueAfter: r.tokenCheckRefreshOrRotationInterval(),

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller_test.go‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1441,6 +1441,155 @@ func TestBootstrapTokenRotationMachinePool(t *testing.T) {
14411441
g.Expect(foundNew).To(BeTrue())
14421442
}
14431443

1444+
func TestBootstrapTokenRefreshIfTokenSecretCleaned(t *testing.T) {
1445+
t.Run("should not recreate the token for Machines", func(t *testing.T) {
1446+
g := NewWithT(t)
1447+
1448+
cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
1449+
cluster.Status.InfrastructureReady = true
1450+
conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
1451+
cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
1452+
1453+
controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
1454+
initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
1455+
1456+
addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
1457+
1458+
workerMachine := newWorkerMachineForCluster(cluster)
1459+
workerJoinConfig := newWorkerJoinKubeadmConfig(metav1.NamespaceDefault, "worker-join-cfg")
1460+
addKubeadmConfigToMachine(workerJoinConfig, workerMachine)
1461+
objects := []client.Object{
1462+
cluster,
1463+
workerMachine,
1464+
workerJoinConfig,
1465+
}
1466+
1467+
objects = append(objects, createSecrets(t, cluster, initConfig)...)
1468+
myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}).Build()
1469+
remoteClient := fake.NewClientBuilder().Build()
1470+
k := &KubeadmConfigReconciler{
1471+
Client: myclient,
1472+
SecretCachingClient: myclient,
1473+
KubeadmInitLock: &myInitLocker{},
1474+
TokenTTL: DefaultTokenTTL,
1475+
ClusterCache: clustercache.NewFakeClusterCache(remoteClient, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
1476+
}
1477+
request := ctrl.Request{
1478+
NamespacedName: client.ObjectKey{
1479+
Namespace: metav1.NamespaceDefault,
1480+
Name: "worker-join-cfg",
1481+
},
1482+
}
1483+
result, err := k.Reconcile(ctx, request)
1484+
g.Expect(err).ToNot(HaveOccurred())
1485+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1486+
1487+
cfg, err := getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
1488+
g.Expect(err).ToNot(HaveOccurred())
1489+
g.Expect(cfg.Status.Ready).To(BeTrue())
1490+
g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
1491+
g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
1492+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1493+
firstToken := cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
1494+
1495+
l := &corev1.SecretList{}
1496+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1497+
g.Expect(l.Items).To(HaveLen(1))
1498+
1499+
t.Log("Token should not get recreated for single Machine since it will not use the new token if spec.bootstrap.dataSecretName was already set")
1500+
1501+
// Simulate token cleaner of Kubernetes having deleted the token secret
1502+
err = remoteClient.Delete(ctx, &l.Items[0])
1503+
g.Expect(err).ToNot(HaveOccurred())
1504+
1505+
result, err = k.Reconcile(ctx, request)
1506+
g.Expect(err).To(HaveOccurred())
1507+
g.Expect(err.Error()).To(ContainSubstring("failed to get bootstrap token secret in order to refresh it"))
1508+
// New token should not have been created
1509+
cfg, err = getKubeadmConfig(myclient, "worker-join-cfg", metav1.NamespaceDefault)
1510+
g.Expect(err).ToNot(HaveOccurred())
1511+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).To(Equal(firstToken))
1512+
1513+
l = &corev1.SecretList{}
1514+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1515+
g.Expect(l.Items).To(BeEmpty())
1516+
})
1517+
t.Run("should recreate the token for MachinePools", func(t *testing.T) {
1518+
_ = feature.MutableGates.Set("MachinePool=true")
1519+
g := NewWithT(t)
1520+
1521+
cluster := builder.Cluster(metav1.NamespaceDefault, "cluster").Build()
1522+
cluster.Status.InfrastructureReady = true
1523+
conditions.MarkTrue(cluster, clusterv1.ControlPlaneInitializedCondition)
1524+
cluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{Host: "100.105.150.1", Port: 6443}
1525+
1526+
controlPlaneInitMachine := newControlPlaneMachine(cluster, "control-plane-init-machine")
1527+
initConfig := newControlPlaneInitKubeadmConfig(controlPlaneInitMachine.Namespace, "control-plane-init-config")
1528+
1529+
addKubeadmConfigToMachine(initConfig, controlPlaneInitMachine)
1530+
1531+
workerMachinePool := newWorkerMachinePoolForCluster(cluster)
1532+
workerJoinConfig := newWorkerJoinKubeadmConfig(workerMachinePool.Namespace, "workerpool-join-cfg")
1533+
addKubeadmConfigToMachinePool(workerJoinConfig, workerMachinePool)
1534+
objects := []client.Object{
1535+
cluster,
1536+
workerMachinePool,
1537+
workerJoinConfig,
1538+
}
1539+
1540+
objects = append(objects, createSecrets(t, cluster, initConfig)...)
1541+
myclient := fake.NewClientBuilder().WithObjects(objects...).WithStatusSubresource(&bootstrapv1.KubeadmConfig{}, &expv1.MachinePool{}).Build()
1542+
remoteClient := fake.NewClientBuilder().Build()
1543+
k := &KubeadmConfigReconciler{
1544+
Client: myclient,
1545+
SecretCachingClient: myclient,
1546+
KubeadmInitLock: &myInitLocker{},
1547+
TokenTTL: DefaultTokenTTL,
1548+
ClusterCache: clustercache.NewFakeClusterCache(remoteClient, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}),
1549+
}
1550+
request := ctrl.Request{
1551+
NamespacedName: client.ObjectKey{
1552+
Namespace: metav1.NamespaceDefault,
1553+
Name: "workerpool-join-cfg",
1554+
},
1555+
}
1556+
result, err := k.Reconcile(ctx, request)
1557+
g.Expect(err).ToNot(HaveOccurred())
1558+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1559+
1560+
cfg, err := getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault)
1561+
g.Expect(err).ToNot(HaveOccurred())
1562+
g.Expect(cfg.Status.Ready).To(BeTrue())
1563+
g.Expect(cfg.Status.DataSecretName).NotTo(BeNil())
1564+
g.Expect(cfg.Status.ObservedGeneration).NotTo(BeNil())
1565+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1566+
firstToken := cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token
1567+
1568+
l := &corev1.SecretList{}
1569+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1570+
g.Expect(l.Items).To(HaveLen(1))
1571+
1572+
t.Log("Ensure that the token gets recreated if it was cleaned up by Kubernetes (e.g. on expiry)")
1573+
1574+
// Simulate token cleaner of Kubernetes having deleted the token secret
1575+
err = remoteClient.Delete(ctx, &l.Items[0])
1576+
g.Expect(err).ToNot(HaveOccurred())
1577+
1578+
result, err = k.Reconcile(ctx, request)
1579+
g.Expect(err).ToNot(HaveOccurred())
1580+
g.Expect(result.RequeueAfter).To(Equal(k.TokenTTL / 3))
1581+
// New token should have been created
1582+
cfg, err = getKubeadmConfig(myclient, "workerpool-join-cfg", metav1.NamespaceDefault)
1583+
g.Expect(err).ToNot(HaveOccurred())
1584+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(BeEmpty())
1585+
g.Expect(cfg.Spec.JoinConfiguration.Discovery.BootstrapToken.Token).ToNot(Equal(firstToken))
1586+
1587+
l = &corev1.SecretList{}
1588+
g.Expect(remoteClient.List(ctx, l, client.ListOption(client.InNamespace(metav1.NamespaceSystem)))).To(Succeed())
1589+
g.Expect(l.Items).To(HaveLen(1))
1590+
})
1591+
}
1592+
14441593
// Ensure the discovery portion of the JoinConfiguration gets generated correctly.
14451594
func TestKubeadmConfigReconciler_Reconcile_DiscoveryReconcileBehaviors(t *testing.T) {
14461595
caHash := []string{"...."}

0 commit comments

Comments
 (0)