Skip to content

Commit 5b2b68f

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #4069 from Skarlso/fix_cloudformation_reconcile
chore(ref): add more robustness to cloudformation stack create in tests
2 parents a690ae5 + b7e4f9a commit 5b2b68f

File tree

3 files changed

+88
-30
lines changed

3 files changed

+88
-30
lines changed

‎cmd/clusterawsadm/cloudformation/service/service.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,34 @@ func (s *Service) ReconcileBootstrapStack(stackName string, t go_cfn.Template, t
8282
return nil
8383
}
8484

85+
func (s *Service) ReconcileBootstrapNoUpdate(stackName string, t go_cfn.Template, tags map[string]string) error {
86+
yaml, err := t.YAML()
87+
processedYaml := string(yaml)
88+
if err != nil {
89+
return errors.Wrap(err, "failed to generate AWS CloudFormation YAML")
90+
}
91+
92+
stackTags := []*cfn.Tag{}
93+
for k, v := range tags {
94+
stackTags = append(stackTags, &cfn.Tag{
95+
Key: aws.String(k),
96+
Value: aws.String(v),
97+
})
98+
}
99+
//nolint:nestif
100+
if err := s.createStack(stackName, processedYaml, stackTags, true); err != nil {
101+
if code, _ := awserrors.Code(errors.Cause(err)); code == "AlreadyExistsException" {
102+
desInput := &cfn.DescribeStacksInput{StackName: aws.String(stackName)}
103+
if err := s.CFN.WaitUntilStackCreateComplete(desInput); err != nil {
104+
return errors.Wrap(err, "failed to wait for AWS CloudFormation stack to be CreateComplete")
105+
}
106+
return nil
107+
}
108+
return err
109+
}
110+
return nil
111+
}
112+
85113
func (s *Service) createStack(stackName, yaml string, tags []*cfn.Tag, deleteOnFailure bool) error {
86114
input := &cfn.CreateStackInput{
87115
Capabilities: aws.StringSlice([]string{cfn.CapabilityCapabilityIam, cfn.CapabilityCapabilityNamedIam}),
@@ -100,7 +128,7 @@ func (s *Service) createStack(stackName, yaml string, tags []*cfn.Tag, deleteOnF
100128
desInput := &cfn.DescribeStacksInput{StackName: aws.String(stackName)}
101129
klog.V(2).Infof("waiting for stack %q to create", stackName)
102130
if err := s.CFN.WaitUntilStackCreateComplete(desInput); err != nil {
103-
return errors.Wrap(err, "failed to create AWS CloudFormation stack")
131+
return errors.Wrap(err, "failed to wait for AWS CloudFormation stack to be CreateComplete")
104132
}
105133

106134
klog.V(2).Infof("stack %q created", stackName)

‎test/e2e/shared/aws.go

Lines changed: 51 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -406,37 +406,65 @@ func NewAWSSessionWithKey(accessKey *iam.AccessKey) client.ConfigProvider {
406406
// createCloudFormationStack ensures the cloudformation stack is up to date.
407407
func createCloudFormationStack(prov client.ConfigProvider, t *cfn_bootstrap.Template, tags map[string]string) error {
408408
By(fmt.Sprintf("Creating AWS CloudFormation stack for AWS IAM resources: stack-name=%s", t.Spec.StackName))
409-
CFN := cfn.New(prov)
410-
cfnSvc := cloudformation.NewService(CFN)
411-
412-
Eventually(func() bool {
413-
err := cfnSvc.ReconcileBootstrapStack(t.Spec.StackName, *renderCustomCloudFormation(t), tags, true)
414-
if err != nil {
415-
By(fmt.Sprintf("Error reconciling Cloud formation stack %v", err))
409+
cfnClient := cfn.New(prov)
410+
// CloudFormation stack will clean up on a failure, we don't need an Eventually here.
411+
// The `create` already does a WaitUntilStackCreateComplete.
412+
cfnSvc := cloudformation.NewService(cfnClient)
413+
err := cfnSvc.ReconcileBootstrapNoUpdate(t.Spec.StackName, *renderCustomCloudFormation(t), tags)
414+
if err != nil {
415+
By(fmt.Sprintf("Error reconciling Cloud formation stack %v", err))
416+
spewCloudFormationResources(cfnClient, t)
417+
418+
stack, derr := cfnClient.DescribeStacks(&cfn.DescribeStacksInput{StackName: aws.String(t.Spec.StackName)})
419+
if derr == nil && len(stack.Stacks) > 0 {
420+
if aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackFailed ||
421+
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackComplete ||
422+
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackInProgress ||
423+
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusCreateFailed ||
424+
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusDeleteFailed {
425+
// If cloudformation stack creation fails due to resources that already exist, stack stays in rollback status and must be manually deleted.
426+
// Delete resources that failed because they already exists.
427+
By("Starting cleanup process as the stack failed to create")
428+
deleteMultitenancyRoles(prov)
429+
deleteResourcesInCloudFormation(prov, t)
430+
}
416431
}
417-
output, err1 := CFN.DescribeStackEvents(&cfn.DescribeStackEventsInput{StackName: aws.String(t.Spec.StackName), NextToken: aws.String("1")})
432+
return err
433+
}
434+
435+
spewCloudFormationResources(cfnClient, t)
436+
return err
437+
}
438+
439+
func spewCloudFormationResources(cfnClient *cfn.CloudFormation, t *cfn_bootstrap.Template) {
440+
output, err := cfnClient.DescribeStackEvents(&cfn.DescribeStackEventsInput{StackName: aws.String(t.Spec.StackName), NextToken: aws.String("1")})
441+
if err != nil {
442+
By(fmt.Sprintf("Error describin Cloud formation stack events %v, skipping", err))
443+
} else {
418444
By("========= Stack Event Output Begin =========")
419445
for _, event := range output.StackEvents {
420446
By(fmt.Sprintf("Event details for %s : Resource: %s, Status: %s, Reason: %s", aws.StringValue(event.LogicalResourceId), aws.StringValue(event.ResourceType), aws.StringValue(event.ResourceStatus), aws.StringValue(event.ResourceStatusReason)))
421447
}
422448
By("========= Stack Event Output End =========")
423-
return err == nil && err1 == nil
424-
}, 2*time.Minute).Should(Equal(true))
425-
426-
stack, err := CFN.DescribeStacks(&cfn.DescribeStacksInput{StackName: aws.String(t.Spec.StackName)})
427-
if err == nil && len(stack.Stacks) > 0 {
428-
deleteMultitenancyRoles(prov)
429-
if aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackFailed ||
430-
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackComplete ||
431-
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackInProgress ||
432-
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusCreateFailed ||
433-
aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusDeleteFailed {
434-
// If cloudformation stack creation fails due to resources that already exist, stack stays in rollback status and must be manually deleted.
435-
// Delete resources that failed because they already exists.
436-
deleteResourcesInCloudFormation(prov, t)
449+
}
450+
out, err := cfnClient.DescribeStackResources(&cfn.DescribeStackResourcesInput{
451+
StackName: aws.String(t.Spec.StackName),
452+
})
453+
if err != nil {
454+
By(fmt.Sprintf("Error describing Stack Resources %v, skipping", err))
455+
} else {
456+
By("========= Stack Resources Output Begin =========")
457+
By("Resource\tType\tStatus")
458+
459+
for _, r := range out.StackResources {
460+
By(fmt.Sprintf("%s\t%s\t%s\t%s",
461+
aws.StringValue(r.ResourceType),
462+
aws.StringValue(r.PhysicalResourceId),
463+
aws.StringValue(r.ResourceStatus),
464+
aws.StringValue(r.ResourceStatusReason)))
437465
}
466+
By("========= Stack Resources Output End =========")
438467
}
439-
return err
440468
}
441469

442470
func SetMultitenancyEnvVars(prov client.ConfigProvider) error {

‎test/e2e/shared/suite.go

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -130,12 +130,14 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
130130
e2eCtx.CloudFormationTemplate = renderCustomCloudFormation(bootstrapTemplate)
131131

132132
if !e2eCtx.Settings.SkipCloudFormationCreation {
133-
err = createCloudFormationStack(e2eCtx.AWSSession, bootstrapTemplate, bootstrapTags)
134-
if err != nil {
135-
deleteCloudFormationStack(e2eCtx.AWSSession, bootstrapTemplate)
136-
err = createCloudFormationStack(e2eCtx.AWSSession, bootstrapTemplate, bootstrapTags)
137-
Expect(err).NotTo(HaveOccurred())
138-
}
133+
Eventually(func(gomega Gomega) bool {
134+
success := true
135+
if err := createCloudFormationStack(e2eCtx.AWSSession, bootstrapTemplate, bootstrapTags); err != nil {
136+
deleteCloudFormationStack(e2eCtx.AWSSession, bootstrapTemplate)
137+
success = false
138+
}
139+
return success
140+
}, 10*time.Minute, 5*time.Second).Should(BeTrue())
139141
}
140142

141143
ensureStackTags(e2eCtx.AWSSession, bootstrapTemplate.Spec.StackName, bootstrapTags)

0 commit comments

Comments
 (0)