fix: cleanup AWS CloudFormation stack in Test environment (kubernetes-sigs#4059)

* Cleanup AWS CloudFormation stack in Test environment

* add security group name to the output

Author: Skarlso

cmd/clusterawsadm/cloudformation/service/service.go

@@ -49,7 +49,7 @@ func NewService(i cloudformationiface.CloudFormationAPI) *Service {
 }
 
 // ReconcileBootstrapStack creates or updates bootstrap CloudFormation.
-func (s *Service) ReconcileBootstrapStack(stackName string, t go_cfn.Template, tags map[string]string) error {
+func (s *Service) ReconcileBootstrapStack(stackName string, t go_cfn.Template, tags map[string]string, deleteOnFailure bool) error {
 	yaml, err := t.YAML()
 	processedYaml := string(yaml)
 	if err != nil {
@@ -64,7 +64,7 @@ func (s *Service) ReconcileBootstrapStack(stackName string, t go_cfn.Template, t
 		})
 	}
 	//nolint:nestif
-	if err := s.createStack(stackName, processedYaml, stackTags); err != nil {
+	if err := s.createStack(stackName, processedYaml, stackTags, deleteOnFailure); err != nil {
 		if code, _ := awserrors.Code(errors.Cause(err)); code == "AlreadyExistsException" {
 			klog.Infof("AWS Cloudformation stack %q already exists, updating", klog.KRef("", stackName))
 			updateErr := s.updateStack(stackName, processedYaml, stackTags)
@@ -82,13 +82,16 @@ func (s *Service) ReconcileBootstrapStack(stackName string, t go_cfn.Template, t
 	return nil
 }
 
-func (s *Service) createStack(stackName, yaml string, tags []*cfn.Tag) error {
+func (s *Service) createStack(stackName, yaml string, tags []*cfn.Tag, deleteOnFailure bool) error {
 	input := &cfn.CreateStackInput{
 		Capabilities: aws.StringSlice([]string{cfn.CapabilityCapabilityIam, cfn.CapabilityCapabilityNamedIam}),
 		TemplateBody: aws.String(yaml),
 		StackName:    aws.String(stackName),
 		Tags:         tags,
 	}
+	if deleteOnFailure {
+		input.OnFailure = aws.String(cfn.OnFailureDelete)
+	}
 	klog.V(2).Infof("creating AWS CloudFormation stack %q", stackName)
 	if _, err := s.CFN.CreateStack(input); err != nil {
 		return errors.Wrap(err, "failed to create AWS CloudFormation stack")

cmd/clusterawsadm/cmd/bootstrap/iam/cloudformation.go

@@ -109,7 +109,7 @@ func createCloudFormationStackCmd() *cobra.Command {
 
 			cfnSvc := cloudformation.NewService(cfn.New(sess))
 
-			err = cfnSvc.ReconcileBootstrapStack(t.Spec.StackName, *t.RenderCloudFormation(), t.Spec.StackTags)
+			err = cfnSvc.ReconcileBootstrapStack(t.Spec.StackName, *t.RenderCloudFormation(), t.Spec.StackTags, false)
 			if err != nil {
 				fmt.Printf("Error: %v\n", err)
 				return err

pkg/cloud/services/securitygroup/securitygroups.go

@@ -305,8 +305,8 @@ func (s *Service) deleteSecurityGroup(sg *infrav1.SecurityGroup, typ string) err
 	}
 
 	if _, err := s.EC2Client.DeleteSecurityGroup(input); awserrors.IsIgnorableSecurityGroupError(err) != nil {
-		record.Warnf(s.scope.InfraCluster(), "FailedDeleteSecurityGroup", "Failed to delete %s SecurityGroup %q: %v", typ, sg.ID, err)
-		return errors.Wrapf(err, "failed to delete security group %q", sg.ID)
+		record.Warnf(s.scope.InfraCluster(), "FailedDeleteSecurityGroup", "Failed to delete %s SecurityGroup %q with name %q: %v", typ, sg.ID, sg.Name, err)
+		return errors.Wrapf(err, "failed to delete security group %q with name %q", sg.ID, sg.Name)
 	}
 
 	record.Eventf(s.scope.InfraCluster(), "SuccessfulDeleteSecurityGroup", "Deleted %s SecurityGroup %q", typ, sg.ID)

test/e2e/shared/aws.go

@@ -410,19 +410,24 @@ func createCloudFormationStack(prov client.ConfigProvider, t *cfn_bootstrap.Temp
 	cfnSvc := cloudformation.NewService(CFN)
 
 	Eventually(func() bool {
-		err := cfnSvc.ReconcileBootstrapStack(t.Spec.StackName, *renderCustomCloudFormation(t), tags)
+		err := cfnSvc.ReconcileBootstrapStack(t.Spec.StackName, *renderCustomCloudFormation(t), tags, true)
 		output, err1 := CFN.DescribeStackEvents(&cfn.DescribeStackEventsInput{StackName: aws.String(t.Spec.StackName), NextToken: aws.String("1")})
+		By("========= Stack Event Output Begin =========")
 		for _, event := range output.StackEvents {
 			By(fmt.Sprintf("Event details for %s : Resource: %s, Status: %s, Reason: %s", aws.StringValue(event.LogicalResourceId), aws.StringValue(event.ResourceType), aws.StringValue(event.ResourceStatus), aws.StringValue(event.ResourceStatusReason)))
 		}
+		By("========= Stack Event Output End =========")
 		return err == nil && err1 == nil
 	}, 2*time.Minute).Should(Equal(true))
+
 	stack, err := CFN.DescribeStacks(&cfn.DescribeStacksInput{StackName: aws.String(t.Spec.StackName)})
 	if err == nil && len(stack.Stacks) > 0 {
 		deleteMultitenancyRoles(prov)
 		if aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackFailed ||
 			aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackComplete ||
-			aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackInProgress {
+			aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusRollbackInProgress ||
+			aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusCreateFailed ||
+			aws.StringValue(stack.Stacks[0].StackStatus) == cfn.StackStatusDeleteFailed {
 			// If cloudformation stack creation fails due to resources that already exist, stack stays in rollback status and must be manually deleted.
 			// Delete resources that failed because they already exists.
 			deleteResourcesInCloudFormation(prov, t)
@@ -445,17 +450,23 @@ func deleteResourcesInCloudFormation(prov client.ConfigProvider, t *cfn_bootstra
 	iamSvc := iam.New(prov)
 	temp := *renderCustomCloudFormation(t)
 	for _, val := range temp.Resources {
+		By(fmt.Sprintf("deleting the following resource: %s", val.AWSCloudFormationType()))
 		tayp := val.AWSCloudFormationType()
 		if tayp == configservice.ResourceTypeAwsIamRole {
 			role := val.(*cfn_iam.Role)
+			By(fmt.Sprintf("cleanup for role with name '%s'", role.RoleName))
 			Eventually(func(gomega Gomega) bool {
 				_, err := iamSvc.DeleteRole(&iam.DeleteRoleInput{RoleName: aws.String(role.RoleName)})
 				return awserrors.IsNotFound(err) || err == nil
 			}, 5*time.Minute, 5*time.Second).Should(BeTrue())
 		}
 		if val.AWSCloudFormationType() == "AWS::IAM::InstanceProfile" {
 			profile := val.(*cfn_iam.InstanceProfile)
-			_, _ = iamSvc.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String(profile.InstanceProfileName)})
+			By(fmt.Sprintf("cleanup for profile with name '%s'", profile.InstanceProfileName))
+			Eventually(func(gomega Gomega) bool {
+				_, err := iamSvc.DeleteInstanceProfile(&iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String(profile.InstanceProfileName)})
+				return awserrors.IsNotFound(err) || err == nil
+			}, 5*time.Minute, 5*time.Second).Should(BeTrue())
 		}
 		if val.AWSCloudFormationType() == "AWS::IAM::ManagedPolicy" {
 			policy := val.(*cfn_iam.ManagedPolicy)
@@ -464,7 +475,12 @@ func deleteResourcesInCloudFormation(prov client.ConfigProvider, t *cfn_bootstra
 			if len(policies.Policies) > 0 {
 				for _, p := range policies.Policies {
 					if aws.StringValue(p.PolicyName) == policy.ManagedPolicyName {
-						_, _ = iamSvc.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: p.Arn})
+						By(fmt.Sprintf("cleanup for policy '%s'", p.String()))
+						Eventually(func(gomega Gomega) bool {
+							_, err := iamSvc.DeletePolicy(&iam.DeletePolicyInput{PolicyArn: p.Arn})
+							return awserrors.IsNotFound(err) || err == nil
+						}, 5*time.Minute, 5*time.Second).Should(BeTrue())
+						// TODO: why is there a break here? Don't we want to clean up everything?
 						break
 					}
 				}