Trigger machine pool instance refresh (node rollout) if bootstrap config reference changes

Author: AndiDog

api/v1beta2/tags.go

@@ -190,6 +190,11 @@ const (
 
 	// MachineNameTagKey is the key for machine name.
 	MachineNameTagKey = "MachineName"
+
+	// LaunchTemplateBootstrapDataSecret is the tag we use to store the `<namespace>/<name>`
+	// of the bootstrap secret that was used to create the user data for the latest launch
+	// template version.
+	LaunchTemplateBootstrapDataSecret = NameAWSProviderPrefix + "bootstrap-data-secret"
 )
 
 // ClusterTagKey generates the key for resources associated with a cluster.

exp/controllers/awsmachinepool_controller.go

@@ -61,6 +61,7 @@ type AWSMachinePoolReconciler struct {
 	WatchFilterValue             string
 	asgServiceFactory            func(cloud.ClusterScoper) services.ASGInterface
 	ec2ServiceFactory            func(scope.EC2Scope) services.EC2Interface
+	reconcileServiceFactory      func(scope.EC2Scope) services.MachinePoolReconcileInterface
 	TagUnmanagedNetworkResources bool
 }
 
@@ -79,6 +80,14 @@ func (r *AWSMachinePoolReconciler) getEC2Service(scope scope.EC2Scope) services.
 	return ec2.NewService(scope)
 }
 
+func (r *AWSMachinePoolReconciler) getReconcileService(scope scope.EC2Scope) services.MachinePoolReconcileInterface {
+	if r.reconcileServiceFactory != nil {
+		return r.reconcileServiceFactory(scope)
+	}
+
+	return ec2.NewService(scope)
+}
+
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachinepools,verbs=get;list;watch;update;patch;delete
 // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachinepools/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machinepools;machinepools/status,verbs=get;list;watch;patch
@@ -227,6 +236,7 @@ func (r *AWSMachinePoolReconciler) reconcileNormal(ctx context.Context, machineP
 
 	ec2Svc := r.getEC2Service(ec2Scope)
 	asgsvc := r.getASGService(clusterScope)
+	reconSvc := r.getReconcileService(ec2Scope)
 
 	// Find existing ASG
 	asg, err := r.findASG(machinePoolScope, asgsvc)
@@ -269,7 +279,7 @@ func (r *AWSMachinePoolReconciler) reconcileNormal(ctx context.Context, machineP
 		machinePoolScope.Info("starting instance refresh", "number of instances", machinePoolScope.MachinePool.Spec.Replicas)
 		return asgsvc.StartASGInstanceRefresh(machinePoolScope)
 	}
-	if err := ec2Svc.ReconcileLaunchTemplate(machinePoolScope, canUpdateLaunchTemplate, runPostLaunchTemplateUpdateOperation); err != nil {
+	if err := reconSvc.ReconcileLaunchTemplate(machinePoolScope, ec2Svc, canUpdateLaunchTemplate, runPostLaunchTemplateUpdateOperation); err != nil {
 		r.Recorder.Eventf(machinePoolScope.AWSMachinePool, corev1.EventTypeWarning, "FailedLaunchTemplateReconcile", "Failed to reconcile launch template: %v", err)
 		machinePoolScope.Error(err, "failed to reconcile launch template")
 		return err
@@ -317,7 +327,7 @@ func (r *AWSMachinePoolReconciler) reconcileNormal(ctx context.Context, machineP
 			ResourceService: asgsvc,
 		},
 	}
-	err = ec2Svc.ReconcileTags(machinePoolScope, resourceServiceToUpdate)
+	err = reconSvc.ReconcileTags(machinePoolScope, resourceServiceToUpdate)
 	if err != nil {
 		return errors.Wrap(err, "error updating tags")
 	}
@@ -378,7 +388,7 @@ func (r *AWSMachinePoolReconciler) reconcileDelete(machinePoolScope *scope.Machi
 	}
 
 	launchTemplateID := machinePoolScope.AWSMachinePool.Status.LaunchTemplateID
-	launchTemplate, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.LaunchTemplateName())
+	launchTemplate, _, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.LaunchTemplateName())
 	if err != nil {
 		return err
 	}
@@ -422,7 +432,7 @@ func (r *AWSMachinePoolReconciler) updatePool(machinePoolScope *scope.MachinePoo
 
 	asgDiff := diffASG(machinePoolScope, existingASG)
 	if asgDiff != "" {
-		machinePoolScope.Debug("asg diff detected", "diff", subnetDiff)
+		machinePoolScope.Debug("asg diff detected", "asgDiff", asgDiff, "subnetDiff", subnetDiff)
 	}
 	if asgDiff != "" || subnetDiff != "" {
 		machinePoolScope.Info("updating AutoScalingGroup")

exp/controllers/awsmachinepool_controller_test.go

@@ -207,6 +207,7 @@ func (r *AWSManagedMachinePoolReconciler) reconcileNormal(
 
 	ekssvc := eks.NewNodegroupService(machinePoolScope)
 	ec2svc := r.getEC2Service(ec2Scope)
+	reconSvc := r.getReconcileService(ec2Scope)
 
 	if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil {
 		canUpdateLaunchTemplate := func() (bool, error) {
@@ -215,7 +216,7 @@ func (r *AWSManagedMachinePoolReconciler) reconcileNormal(
 		runPostLaunchTemplateUpdateOperation := func() error {
 			return nil
 		}
-		if err := ec2svc.ReconcileLaunchTemplate(machinePoolScope, canUpdateLaunchTemplate, runPostLaunchTemplateUpdateOperation); err != nil {
+		if err := reconSvc.ReconcileLaunchTemplate(machinePoolScope, ec2svc, canUpdateLaunchTemplate, runPostLaunchTemplateUpdateOperation); err != nil {
 			r.Recorder.Eventf(machinePoolScope.ManagedMachinePool, corev1.EventTypeWarning, "FailedLaunchTemplateReconcile", "Failed to reconcile launch template: %v", err)
 			machinePoolScope.Error(err, "failed to reconcile launch template")
 			conditions.MarkFalse(machinePoolScope.ManagedMachinePool, expinfrav1.LaunchTemplateReadyCondition, expinfrav1.LaunchTemplateReconcileFailedReason, clusterv1.ConditionSeverityError, "")
@@ -227,7 +228,7 @@ func (r *AWSManagedMachinePoolReconciler) reconcileNormal(
 			ResourceID:      &launchTemplateID,
 			ResourceService: ec2svc,
 		}}
-		if err := ec2svc.ReconcileTags(machinePoolScope, resourceServiceToUpdate); err != nil {
+		if err := reconSvc.ReconcileTags(machinePoolScope, resourceServiceToUpdate); err != nil {
 			return errors.Wrap(err, "error updating tags")
 		}
 
@@ -258,7 +259,7 @@ func (r *AWSManagedMachinePoolReconciler) reconcileDelete(
 
 	if machinePoolScope.ManagedMachinePool.Spec.AWSLaunchTemplate != nil {
 		launchTemplateID := machinePoolScope.ManagedMachinePool.Status.LaunchTemplateID
-		launchTemplate, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.LaunchTemplateName())
+		launchTemplate, _, _, err := ec2Svc.GetLaunchTemplate(machinePoolScope.LaunchTemplateName())
 		if err != nil {
 			return err
 		}
@@ -347,3 +348,7 @@ func managedControlPlaneToManagedMachinePoolMapFunc(c client.Client, gvk schema.
 func (r *AWSManagedMachinePoolReconciler) getEC2Service(scope scope.EC2Scope) services.EC2Interface {
 	return ec2.NewService(scope)
 }
+
+func (r *AWSManagedMachinePoolReconciler) getReconcileService(scope scope.EC2Scope) services.MachinePoolReconcileInterface {
+	return ec2.NewService(scope)
+}

exp/controllers/awsmanagedmachinepool_controller.go

@@ -18,6 +18,7 @@ package scope
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
@@ -36,7 +37,7 @@ type LaunchTemplateScope interface {
 	SetLaunchTemplateIDStatus(id string)
 	GetLaunchTemplateLatestVersionStatus() string
 	SetLaunchTemplateLatestVersionStatus(version string)
-	GetRawBootstrapData() ([]byte, error)
+	GetRawBootstrapData() ([]byte, *types.NamespacedName, error)
 
 	IsEKSManaged() bool
 	AdditionalTags() infrav1.Tags

pkg/cloud/scope/launchtemplate.go

@@ -131,36 +131,32 @@ func (m *MachinePoolScope) Namespace() string {
 	return m.AWSMachinePool.Namespace
 }
 
-// GetRawBootstrapData returns the bootstrap data from the secret in the Machine's bootstrap.dataSecretName.
-// todo(rudoi): stolen from MachinePool - any way to reuse?
-func (m *MachinePoolScope) GetRawBootstrapData() ([]byte, error) {
-	data, _, err := m.getBootstrapData()
+// GetRawBootstrapData returns the bootstrap data from the secret in the Machine's bootstrap.dataSecretName,
+// including the secret's namespaced name.
+func (m *MachinePoolScope) GetRawBootstrapData() ([]byte, *types.NamespacedName, error) {
+	data, _, bootstrapDataSecretKey, err := m.getBootstrapData()
 
-	return data, err
+	return data, bootstrapDataSecretKey, err
 }
 
-func (m *MachinePoolScope) GetRawBootstrapDataWithFormat() ([]byte, string, error) {
-	return m.getBootstrapData()
-}
-
-func (m *MachinePoolScope) getBootstrapData() ([]byte, string, error) {
+func (m *MachinePoolScope) getBootstrapData() ([]byte, string, *types.NamespacedName, error) {
 	if m.MachinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil {
-		return nil, "", errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil")
+		return nil, "", nil, errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil")
 	}
 
 	secret := &corev1.Secret{}
 	key := types.NamespacedName{Namespace: m.Namespace(), Name: *m.MachinePool.Spec.Template.Spec.Bootstrap.DataSecretName}
 
 	if err := m.Client.Get(context.TODO(), key, secret); err != nil {
-		return nil, "", errors.Wrapf(err, "failed to retrieve bootstrap data secret %s for AWSMachine %s/%s", key.Name, m.Namespace(), m.Name())
+		return nil, "", nil, errors.Wrapf(err, "failed to retrieve bootstrap data secret %s for AWSMachinePool %s/%s", key.Name, m.Namespace(), m.Name())
 	}
 
 	value, ok := secret.Data["value"]
 	if !ok {
-		return nil, "", errors.New("error retrieving bootstrap data: secret value key is missing")
+		return nil, "", nil, errors.New("error retrieving bootstrap data: secret value key is missing")
 	}
 
-	return value, string(secret.Data["format"]), nil
+	return value, string(secret.Data["format"]), &key, nil
 }
 
 // AdditionalTags merges AdditionalTags from the scope's AWSCluster and AWSMachinePool. If the same key is present in both,

pkg/cloud/scope/machinepool.go

@@ -323,24 +323,24 @@ func (s *ManagedMachinePoolScope) Namespace() string {
 	return s.ManagedMachinePool.Namespace
 }
 
-func (s *ManagedMachinePoolScope) GetRawBootstrapData() ([]byte, error) {
+func (s *ManagedMachinePoolScope) GetRawBootstrapData() ([]byte, *types.NamespacedName, error) {
 	if s.MachinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil {
-		return nil, errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil")
+		return nil, nil, errors.New("error retrieving bootstrap data: linked Machine's bootstrap.dataSecretName is nil")
 	}
 
 	secret := &corev1.Secret{}
 	key := types.NamespacedName{Namespace: s.Namespace(), Name: *s.MachinePool.Spec.Template.Spec.Bootstrap.DataSecretName}
 
 	if err := s.Client.Get(context.TODO(), key, secret); err != nil {
-		return nil, errors.Wrapf(err, "failed to retrieve bootstrap data secret for AWSManagedMachinePool %s/%s", s.Namespace(), s.Name())
+		return nil, nil, errors.Wrapf(err, "failed to retrieve bootstrap data secret for AWSManagedMachinePool %s/%s", s.Namespace(), s.Name())
 	}
 
 	value, ok := secret.Data["value"]
 	if !ok {
-		return nil, errors.New("error retrieving bootstrap data: secret value key is missing")
+		return nil, nil, errors.New("error retrieving bootstrap data: secret value key is missing")
 	}
 
-	return value, nil
+	return value, &key, nil
 }
 
 func (s *ManagedMachinePoolScope) GetObjectMeta() *metav1.ObjectMeta {