roachprod: gc ibm clusters with no tags

Starting with #148008, some partially created clusters (without tags)
get listed for garbage collection, but are never actually deleted
because the `DeleteCluster()` lists VMs to destroy via tags.

This PR updates the `DeleteCluster()` function to also list VMs by
instance names and properly destroy some leftover clusters.

Epic: none
Release note: None

Author: golgeek

pkg/roachprod/vm/ibm/ibm_extended_types.go

@@ -588,6 +588,11 @@ func (i *instance) toVM() vm.VM {
 		if err != nil {
 			vmErrors = append(vmErrors, errors.Wrap(err, "unable to compute lifetime"))
 		}
+	} else {
+		// Missing lifetime tag, use the default lifetime.
+		// This is not an error, but a fallback to ensure the VM has a lifetime
+		// even if the tag is not set to avoid GCing it right away.
+		lifetime = vm.DefaultLifetime
 	}
 
 	privateIP := i.getPrivateIPAddress()

pkg/roachprod/vm/ibm/provider.go

@@ -545,8 +545,16 @@ func (p *Provider) DeleteCluster(l *logger.Logger, name string) error {
 
 	svc := p.getGlobalSearchService()
 
-	// Get the resources with the cluster name tag.
-	query := fmt.Sprintf(`tags:"%s:true" AND "%s:%s"`, vm.TagRoachprod, vm.TagCluster, name)
+	// The IBM API sometimes silently fails on the tagging request and resources
+	// end up being created without any tags.
+	// The query below will look for resources that are properly tagged with the
+	// roachprod and cluster tags, but will also fallback to searching via the
+	// instance name to ensure all instances are properly deleted.
+	query := fmt.Sprintf(
+		`(tags:"%s:true AND %s:%s") OR (NOT (tags:"%s:true") AND name:%s-*)`,
+		vm.TagRoachprod, vm.TagCluster, name,
+		vm.TagRoachprod, name,
+	)
 	searchOptions := svc.NewSearchOptions().SetLimit(defaultPaginationLimit).SetQuery(query)
 
 	instances := make([]*instance, 0)

pkg/roachprod/vm/vm.go

@@ -43,6 +43,8 @@ const (
 	ArchFIPS    = CPUArch("fips")
 	ArchS390x   = CPUArch("s390x")
 	ArchUnknown = CPUArch("unknown")
+
+	DefaultLifetime = 12 * time.Hour
 )
 
 // UnimplementedError is returned when a method is not implemented by a
@@ -308,7 +310,7 @@ type CreateOpts struct {
 func DefaultCreateOpts() CreateOpts {
 	defaultCreateOpts := CreateOpts{
 		ClusterName:    "",
-		Lifetime:       12 * time.Hour,
+		Lifetime:       DefaultLifetime,
 		GeoDistributed: false,
 		VMProviders:    []string{},
 		OsVolumeSize:   10,