Skip to content

Commit a37ebbe

Browse files
invidianDongsu Park
invidian
and
Dongsu Park
committed
controllers: add Ignition support
This commit finalizes addition of Ignition support as bootstrap data format. Co-authored-by: Dongsu Park <[email protected]> Signed-off-by: Mateusz Gozdek <[email protected]>
1 parent 258b768 commit a37ebbe

File tree

12 files changed

+872
-205
lines changed

12 files changed

+872
-205
lines changed

‎controllers/awsmachine_controller.go

Lines changed: 127 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,12 @@ package controllers
1818

1919
import (
2020
"context"
21+
"encoding/json"
2122
"fmt"
2223
"reflect"
2324

2425
"github.com/aws/aws-sdk-go/aws"
26+
ignTypes "github.com/flatcar-linux/ignition/config/v2_3/types"
2527
"github.com/go-logr/logr"
2628
"github.com/pkg/errors"
2729
corev1 "k8s.io/api/core/v1"
@@ -46,6 +48,7 @@ import (
4648
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/ec2"
4749
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/elb"
4850
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/instancestate"
51+
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/s3"
4952
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/secretsmanager"
5053
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/ssm"
5154
"sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/services/userdata"
@@ -70,6 +73,7 @@ type AWSMachineReconciler struct {
7073
elbServiceFactory func(scope.ELBScope) services.ELBInterface
7174
secretsManagerServiceFactory func(cloud.ClusterScoper) services.SecretInterface
7275
SSMServiceFactory func(cloud.ClusterScoper) services.SecretInterface
76+
objectStoreServiceFactory func(cloud.ClusterScoper) services.ObjectStoreInterface
7377
Endpoints []scope.ServiceEndpoint
7478
WatchFilterValue string
7579
}
@@ -119,6 +123,14 @@ func (r *AWSMachineReconciler) getELBService(elbScope scope.ELBScope) services.E
119123
return elb.NewService(elbScope)
120124
}
121125

126+
func (r *AWSMachineReconciler) getObjectStoreService(scope scope.S3Scope) services.ObjectStoreInterface {
127+
if r.objectStoreServiceFactory != nil {
128+
return r.objectStoreServiceFactory(scope)
129+
}
130+
131+
return s3.NewService(scope)
132+
}
133+
122134
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines,verbs=get;list;watch;create;update;patch;delete
123135
// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=awsmachines/status,verbs=get;update;patch
124136
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=machines;machines/status,verbs=get;list;watch
@@ -197,16 +209,16 @@ func (r *AWSMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request)
197209
switch infraScope := infraCluster.(type) {
198210
case *scope.ManagedControlPlaneScope:
199211
if !awsMachine.ObjectMeta.DeletionTimestamp.IsZero() {
200-
return r.reconcileDelete(machineScope, infraScope, infraScope, nil)
212+
return r.reconcileDelete(machineScope, infraScope, infraScope, nil, nil)
201213
}
202214

203-
return r.reconcileNormal(ctx, machineScope, infraScope, infraScope, nil)
215+
return r.reconcileNormal(ctx, machineScope, infraScope, infraScope, nil, nil)
204216
case *scope.ClusterScope:
205217
if !awsMachine.ObjectMeta.DeletionTimestamp.IsZero() {
206-
return r.reconcileDelete(machineScope, infraScope, infraScope, infraScope)
218+
return r.reconcileDelete(machineScope, infraScope, infraScope, infraScope, infraScope)
207219
}
208220

209-
return r.reconcileNormal(ctx, machineScope, infraScope, infraScope, infraScope)
221+
return r.reconcileNormal(ctx, machineScope, infraScope, infraScope, infraScope, infraScope)
210222
default:
211223
return ctrl.Result{}, errors.New("infraCluster has unknown type")
212224
}
@@ -271,16 +283,14 @@ func (r *AWSMachineReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma
271283
)
272284
}
273285

274-
func (r *AWSMachineReconciler) reconcileDelete(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope, elbScope scope.ELBScope) (ctrl.Result, error) {
286+
func (r *AWSMachineReconciler) reconcileDelete(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope, elbScope scope.ELBScope, objectStoreScope scope.S3Scope) (ctrl.Result, error) {
275287
machineScope.Info("Handling deleted AWSMachine")
276288

277289
ec2Service := r.getEC2Service(ec2Scope)
278290

279-
if machineScope.UseSecretsManager() {
280-
if err := r.deleteEncryptedBootstrapDataSecret(machineScope, clusterScope); err != nil {
281-
machineScope.Error(err, "unable to delete machine")
282-
return ctrl.Result{}, err
283-
}
291+
if err := r.deleteBootstrapData(machineScope, clusterScope, objectStoreScope); err != nil {
292+
machineScope.Error(err, "unable to delete machine")
293+
return ctrl.Result{}, err
284294
}
285295

286296
instance, err := r.findInstance(machineScope, ec2Service)
@@ -418,19 +428,17 @@ func (r *AWSMachineReconciler) findInstance(scope *scope.MachineScope, ec2svc se
418428
return instance, nil
419429
}
420430

421-
func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope, elbScope scope.ELBScope) (ctrl.Result, error) {
431+
func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, ec2Scope scope.EC2Scope, elbScope scope.ELBScope, objectStoreScope scope.S3Scope) (ctrl.Result, error) {
422432
machineScope.Info("Reconciling AWSMachine")
423433

424434
// If the AWSMachine is in an error state, return early.
425435
if machineScope.HasFailed() {
426436
machineScope.Info("Error state detected, skipping reconciliation")
427437

428-
if machineScope.UseSecretsManager() {
429-
// If we are in a failed state, delete the secret regardless of instance state
430-
if err := r.deleteEncryptedBootstrapDataSecret(machineScope, clusterScope); err != nil {
431-
machineScope.Error(err, "unable to reconcile machine")
432-
return ctrl.Result{}, err
433-
}
438+
// If we are in a failed state, delete the secret regardless of instance state.
439+
if err := r.deleteBootstrapData(machineScope, clusterScope, objectStoreScope); err != nil {
440+
machineScope.Error(err, "unable to reconcile machine")
441+
return ctrl.Result{}, err
434442
}
435443

436444
return ctrl.Result{}, nil
@@ -477,7 +485,14 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *
477485
return ctrl.Result{}, patchErr
478486
}
479487
}
480-
instance, err = r.createInstance(ec2svc, machineScope, clusterScope)
488+
489+
var objectStoreSvc services.ObjectStoreInterface
490+
491+
if objectStoreScope != nil {
492+
objectStoreSvc = r.getObjectStoreService(objectStoreScope)
493+
}
494+
495+
instance, err = r.createInstance(ec2svc, machineScope, clusterScope, objectStoreSvc)
481496
if err != nil {
482497
machineScope.Error(err, "unable to create instance")
483498
conditions.MarkFalse(machineScope.AWSMachine, infrav1.InstanceReadyCondition, infrav1.InstanceProvisionFailedReason, clusterv1.ConditionSeverityError, err.Error())
@@ -533,7 +548,7 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *
533548
}
534549

535550
// reconcile the deletion of the bootstrap data secret now that we have updated instance state
536-
if deleteSecretErr := r.deleteEncryptedBootstrapDataSecret(machineScope, clusterScope); deleteSecretErr != nil {
551+
if deleteSecretErr := r.deleteBootstrapData(machineScope, clusterScope, objectStoreScope); deleteSecretErr != nil {
537552
r.Log.Error(deleteSecretErr, "unable to delete secrets")
538553
return ctrl.Result{}, deleteSecretErr
539554
}
@@ -585,10 +600,6 @@ func (r *AWSMachineReconciler) reconcileNormal(_ context.Context, machineScope *
585600
}
586601

587602
func (r *AWSMachineReconciler) deleteEncryptedBootstrapDataSecret(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper) error {
588-
if !machineScope.UseSecretsManager() {
589-
return nil
590-
}
591-
592603
secretSvc, secretBackendErr := r.getSecretService(machineScope, clusterScope)
593604
if secretBackendErr != nil {
594605
machineScope.Error(secretBackendErr, "unable to get secret service backend")
@@ -621,33 +632,41 @@ func (r *AWSMachineReconciler) deleteEncryptedBootstrapDataSecret(machineScope *
621632
return nil
622633
}
623634

624-
func (r *AWSMachineReconciler) createInstance(ec2svc services.EC2Interface, machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper) (*infrav1.Instance, error) {
635+
func (r *AWSMachineReconciler) createInstance(ec2svc services.EC2Interface, machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, objectStoreSvc services.ObjectStoreInterface) (*infrav1.Instance, error) {
625636
machineScope.Info("Creating EC2 instance")
626637

627-
userData, userDataErr := r.resolveUserData(machineScope, clusterScope)
638+
userData, userDataFormat, userDataErr := r.resolveUserData(machineScope, clusterScope, objectStoreSvc)
628639
if userDataErr != nil {
629640
return nil, errors.Wrapf(userDataErr, "failed to resolve userdata")
630641
}
631642

632-
instance, err := ec2svc.CreateInstance(machineScope, userData)
643+
instance, err := ec2svc.CreateInstance(machineScope, userData, userDataFormat)
633644
if err != nil {
634645
return nil, errors.Wrapf(err, "failed to create AWSMachine instance")
635646
}
636647

637648
return instance, nil
638649
}
639650

640-
func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper) ([]byte, error) {
641-
userData, err := machineScope.GetRawBootstrapData()
651+
func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, objectStoreSvc services.ObjectStoreInterface) ([]byte, string, error) {
652+
userData, userDataFormat, err := machineScope.GetRawBootstrapDataWithFormat()
642653
if err != nil {
643654
r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeWarning, "FailedGetBootstrapData", err.Error())
644-
return nil, err
655+
return nil, "", err
656+
}
657+
658+
if machineScope.UseSecretsManager(userDataFormat) {
659+
userData, err = r.cloudInitUserData(machineScope, clusterScope, userData)
645660
}
646661

647-
if !machineScope.UseSecretsManager() {
648-
return userData, nil
662+
if machineScope.UseIgnition(userDataFormat) {
663+
userData, err = r.ignitionUserData(machineScope, objectStoreSvc, userData)
649664
}
650665

666+
return userData, userDataFormat, err
667+
}
668+
669+
func (r *AWSMachineReconciler) cloudInitUserData(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, userData []byte) ([]byte, error) {
651670
secretSvc, secretBackendErr := r.getSecretService(machineScope, clusterScope)
652671
if secretBackendErr != nil {
653672
machineScope.Error(secretBackendErr, "unable to reconcile machine")
@@ -681,6 +700,83 @@ func (r *AWSMachineReconciler) resolveUserData(machineScope *scope.MachineScope,
681700
return encryptedCloudInit, nil
682701
}
683702

703+
func (r *AWSMachineReconciler) ignitionUserData(scope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface, userData []byte) ([]byte, error) {
704+
if objectStoreSvc == nil {
705+
return nil, errors.New("object store service not available")
706+
}
707+
708+
objectURL, err := objectStoreSvc.Create(scope, userData)
709+
if err != nil {
710+
return nil, errors.Wrap(err, "creating userdata object")
711+
}
712+
713+
ignData := &ignTypes.Config{
714+
Ignition: ignTypes.Ignition{
715+
Version: "2.3.0",
716+
Config: ignTypes.IgnitionConfig{
717+
Append: []ignTypes.ConfigReference{
718+
{
719+
Source: objectURL,
720+
},
721+
},
722+
},
723+
},
724+
}
725+
726+
ignitionUserData, err := json.Marshal(ignData)
727+
if err != nil {
728+
r.Recorder.Eventf(scope.AWSMachine, corev1.EventTypeWarning, "FailedGenerateIgnition", err.Error())
729+
return nil, errors.Wrap(err, "serializing generated data")
730+
}
731+
732+
return ignitionUserData, nil
733+
}
734+
735+
func (r *AWSMachineReconciler) deleteBootstrapData(machineScope *scope.MachineScope, clusterScope cloud.ClusterScoper, objectStoreScope scope.S3Scope) error {
736+
if err := r.deleteEncryptedBootstrapDataSecret(machineScope, clusterScope); err != nil {
737+
return err
738+
}
739+
740+
if objectStoreScope != nil {
741+
// Bootstrap data will be removed from S3 if it is already populated.
742+
if err := r.deleteIgnitionBootstrapDataFromS3(machineScope, r.getObjectStoreService(objectStoreScope)); err != nil {
743+
return err
744+
}
745+
}
746+
747+
return nil
748+
}
749+
750+
func (r *AWSMachineReconciler) deleteIgnitionBootstrapDataFromS3(machineScope *scope.MachineScope, objectStoreSvc services.ObjectStoreInterface) error {
751+
// Do nothing if the AWSMachine is not in a failed state, and is operational from an EC2 perspective, but does not have a node reference
752+
if !machineScope.HasFailed() && machineScope.InstanceIsOperational() && machineScope.Machine.Status.NodeRef == nil && !machineScope.AWSMachineIsDeleted() {
753+
return nil
754+
}
755+
756+
// If bootstrap data has not been populated yet, we cannot determine it's format, so there is probably nothing to do.
757+
if machineScope.Machine.Spec.Bootstrap.DataSecretName == nil {
758+
return nil
759+
}
760+
761+
machineScope.Info("Deleting unneeded entry from AWS S3", "secretPrefix", machineScope.GetSecretPrefix())
762+
763+
_, userDataFormat, err := machineScope.GetRawBootstrapDataWithFormat()
764+
if err != nil {
765+
r.Recorder.Eventf(machineScope.AWSMachine, corev1.EventTypeWarning, "FailedGetBootstrapData", err.Error())
766+
return err
767+
}
768+
769+
if !machineScope.UseIgnition(userDataFormat) {
770+
return nil
771+
}
772+
773+
if err := objectStoreSvc.Delete(machineScope); err != nil {
774+
return errors.Wrap(err, "deleting bootstrap data object")
775+
}
776+
777+
return nil
778+
}
779+
684780
func (r *AWSMachineReconciler) reconcileLBAttachment(machineScope *scope.MachineScope, elbScope scope.ELBScope, i *infrav1.Instance) error {
685781
if !machineScope.IsControlPlane() {
686782
return nil

‎controllers/awsmachine_controller_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ func TestAWSMachineReconciler_IntegrationTests(t *testing.T) {
151151
return secretMock
152152
}
153153

154-
_, err = reconciler.reconcileNormal(ctx, ms, cs, cs, cs)
154+
_, err = reconciler.reconcileNormal(ctx, ms, cs, cs, cs, cs)
155155
g.Expect(err).To(BeNil())
156156
expectConditions(g, ms.AWSMachine, []conditionAssertion{{infrav1.SecurityGroupsReadyCondition, corev1.ConditionTrue, "", ""},
157157
{infrav1.InstanceReadyCondition, corev1.ConditionTrue, "", ""},
@@ -207,7 +207,7 @@ func TestAWSMachineReconciler_IntegrationTests(t *testing.T) {
207207
return elbSvc
208208
}
209209

210-
_, err = reconciler.reconcileDelete(ms, cs, cs, cs)
210+
_, err = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
211211
g.Expect(err).To(BeNil())
212212
expectConditions(g, ms.AWSMachine, []conditionAssertion{
213213
{infrav1.InstanceReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason},
@@ -300,7 +300,7 @@ func TestAWSMachineReconciler_IntegrationTests(t *testing.T) {
300300
return secretMock
301301
}
302302

303-
_, err = reconciler.reconcileNormal(ctx, ms, cs, cs, cs)
303+
_, err = reconciler.reconcileNormal(ctx, ms, cs, cs, cs, cs)
304304
g.Expect(err).Should(HaveOccurred())
305305
expectConditions(g, ms.AWSMachine, []conditionAssertion{{infrav1.InstanceReadyCondition, corev1.ConditionTrue, "", ""}})
306306
g.Expect(ms.AWSMachine.Finalizers).Should(ContainElement(infrav1.MachineFinalizer))
@@ -359,7 +359,7 @@ func TestAWSMachineReconciler_IntegrationTests(t *testing.T) {
359359
return elbSvc
360360
}
361361

362-
_, err = reconciler.reconcileDelete(ms, cs, cs, cs)
362+
_, err = reconciler.reconcileDelete(ms, cs, cs, cs, cs)
363363
g.Expect(err).Should(HaveOccurred())
364364
expectConditions(g, ms.AWSMachine, []conditionAssertion{{infrav1.InstanceReadyCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityWarning, "DeletingFailed"},
365365
{infrav1.ELBAttachedCondition, corev1.ConditionFalse, clusterv1.ConditionSeverityInfo, clusterv1.DeletedReason}})

0 commit comments

Comments
 (0)