Merge #150873 #150951

150873: roachprod: gc ibm clusters with no tags r=srosenberg a=golgeek

Starting with #148008, some partially created clusters (without tags) get listed for garbage collection, but are never actually deleted because the `DeleteCluster()` lists VMs to destroy via tags.

This PR updates the `DeleteCluster()` function to also list VMs by instance names and properly destroy some leftover clusters.

Epic: none
Release note: None

150951: auditloggingccl: fix TestReducedAuditConfig r=kyle-a-wong a=kyle-a-wong

Test started flaking due to pebble logs containing matching the regex used to assert that certain audit logs existed.

Adds a more specific regex to assert found augit config logs.

Fixes: #150140
Epic: None
Release note: None

Co-authored-by: Ludovic Leroux <[email protected]>
Co-authored-by: Kyle Wong <[email protected]>

Author: 3 people

pkg/ccl/auditloggingccl/audit_logging_test.go

@@ -389,7 +389,7 @@ func TestReducedAuditConfig(t *testing.T) {
 			0,
 			math.MaxInt64,
 			10000,
-			regexp.MustCompile(stmt),
+			regexp.MustCompile(`"Statement":"`+stmt),
 			log.WithMarkedSensitiveData,
 		)
 

pkg/roachprod/vm/ibm/ibm_extended_types.go

@@ -588,6 +588,11 @@ func (i *instance) toVM() vm.VM {
 		if err != nil {
 			vmErrors = append(vmErrors, errors.Wrap(err, "unable to compute lifetime"))
 		}
+	} else {
+		// Missing lifetime tag, use the default lifetime.
+		// This is not an error, but a fallback to ensure the VM has a lifetime
+		// even if the tag is not set to avoid GCing it right away.
+		lifetime = vm.DefaultLifetime
 	}
 
 	privateIP := i.getPrivateIPAddress()

pkg/roachprod/vm/ibm/provider.go

@@ -545,8 +545,16 @@ func (p *Provider) DeleteCluster(l *logger.Logger, name string) error {
 
 	svc := p.getGlobalSearchService()
 
-	// Get the resources with the cluster name tag.
-	query := fmt.Sprintf(`tags:"%s:true" AND "%s:%s"`, vm.TagRoachprod, vm.TagCluster, name)
+	// The IBM API sometimes silently fails on the tagging request and resources
+	// end up being created without any tags.
+	// The query below will look for resources that are properly tagged with the
+	// roachprod and cluster tags, but will also fallback to searching via the
+	// instance name to ensure all instances are properly deleted.
+	query := fmt.Sprintf(
+		`(tags:"%s:true AND %s:%s") OR (NOT (tags:"%s:true") AND name:%s-*)`,
+		vm.TagRoachprod, vm.TagCluster, name,
+		vm.TagRoachprod, name,
+	)
 	searchOptions := svc.NewSearchOptions().SetLimit(defaultPaginationLimit).SetQuery(query)
 
 	instances := make([]*instance, 0)

pkg/roachprod/vm/vm.go

@@ -43,6 +43,8 @@ const (
 	ArchFIPS    = CPUArch("fips")
 	ArchS390x   = CPUArch("s390x")
 	ArchUnknown = CPUArch("unknown")
+
+	DefaultLifetime = 12 * time.Hour
 )
 
 // UnimplementedError is returned when a method is not implemented by a
@@ -308,7 +310,7 @@ type CreateOpts struct {
 func DefaultCreateOpts() CreateOpts {
 	defaultCreateOpts := CreateOpts{
 		ClusterName:    "",
-		Lifetime:       12 * time.Hour,
+		Lifetime:       DefaultLifetime,
 		GeoDistributed: false,
 		VMProviders:    []string{},
 		OsVolumeSize:   10,