add logic to return successfully on deletion if database instance was already deleted (#453)

Author: markdboyd

services/rds/rds.go

@@ -553,8 +553,7 @@ func (d *dedicatedDBAdapter) waitForDbDeleted(operation base.Operation, i *RDSIn
 	for !isDeleted && attempt <= int(d.settings.PollAwsMaxRetries) {
 		dbState, err = d.checkDBStatus(database)
 		if err != nil {
-			var exception *rdsTypes.DBInstanceNotFoundFault
-			if !errors.As(err, &exception) {
+			if !isDatabaseInstanceNotFoundError(err) {
 				updateErr := jobs.WriteAsyncJobMessage(d.db, i.ServiceID, i.Uuid, operation, base.InstanceNotGone, fmt.Sprintf("Could not check database status: %s", err))
 				if updateErr != nil {
 					err = fmt.Errorf("waitForDbDeleted: while handling error %w, error updating async job message %w", err, updateErr)
@@ -590,7 +589,12 @@ func (d *dedicatedDBAdapter) deleteDatabaseInstance(i *RDSInstance, operation ba
 	params := prepareDeleteDbInput(database)
 	_, err := d.rds.DeleteDBInstance(context.TODO(), params)
 	if err != nil {
-		return fmt.Errorf("deleteDatabaseInstance: %w", err)
+		if isDatabaseInstanceNotFoundError(err) {
+			d.logger.Debug(fmt.Sprintf("database %s was already deleted, continuing", database))
+			return nil
+		} else {
+			return fmt.Errorf("deleteDatabaseInstance: %w", err)
+		}
 	}
 
 	err = d.waitForDbDeleted(operation, i, database)
@@ -666,3 +670,8 @@ func prepareDeleteDbInput(database string) *rds.DeleteDBInstanceInput {
 		SkipFinalSnapshot:      aws.Bool(true),
 	}
 }
+
+func isDatabaseInstanceNotFoundError(err error) bool {
+	var notFoundException *rdsTypes.DBInstanceNotFoundFault
+	return errors.As(err, &notFoundException)
+}

services/rds/rds_test.go

@@ -24,8 +24,6 @@ import (
 	"github.com/google/uuid"
 )
 
-var logger lager.Logger
-
 func NewTestDedicatedDBAdapter(s *config.Settings, db *gorm.DB, rdsClient RDSClientInterface, parameterGroupClient parameterGroupClient) *dedicatedDBAdapter {
 	logger := lager.NewLogger("aws-rds-test")
 	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))
@@ -1901,7 +1899,7 @@ func TestAsyncDeleteDB(t *testing.T) {
 	}
 
 	dbInstanceNotFoundErr := &rdsTypes.DBInstanceNotFoundFault{
-		Message: aws.String("operation failed"),
+		Message: aws.String("not found"),
 	}
 
 	testCases := map[string]struct {
@@ -2055,6 +2053,53 @@ func TestAsyncDeleteDB(t *testing.T) {
 			expectedState:       base.InstanceInProgress,
 			expectedRecordCount: 1,
 		},
+		"database already deleted": {
+			dbAdapter: NewTestDedicatedDBAdapter(
+				&config.Settings{
+					PollAwsRetryDelaySeconds: 0,
+					PollAwsMaxRetries:        1,
+				},
+				brokerDB,
+				&mockRDSClient{
+					deleteDbInstancesErrs: []error{dbInstanceNotFoundErr},
+				},
+				&mockParameterGroupClient{},
+			),
+			dbInstance: &RDSInstance{
+				Instance: base.Instance{
+					Request: request.Request{
+						ServiceID: helpers.RandStr(10),
+					},
+					Uuid: helpers.RandStr(10),
+				},
+				Database: helpers.RandStr(10),
+			},
+			expectedState: base.InstanceGone,
+		},
+		"replica and database already deleted": {
+			dbAdapter: NewTestDedicatedDBAdapter(
+				&config.Settings{
+					PollAwsRetryDelaySeconds: 0,
+					PollAwsMaxRetries:        1,
+				},
+				brokerDB,
+				&mockRDSClient{
+					deleteDbInstancesErrs: []error{dbInstanceNotFoundErr, dbInstanceNotFoundErr},
+				},
+				&mockParameterGroupClient{},
+			),
+			dbInstance: &RDSInstance{
+				Instance: base.Instance{
+					Request: request.Request{
+						ServiceID: helpers.RandStr(10),
+					},
+					Uuid: helpers.RandStr(10),
+				},
+				Database:        helpers.RandStr(10),
+				ReplicaDatabase: helpers.RandStr(10),
+			},
+			expectedState: base.InstanceGone,
+		},
 	}
 
 	for name, test := range testCases {