roachtest: retry invalid descriptors check

This check occasionally times out due to the cluster
being overloaded. Wrap it in a retry loop to reduce
flakes.

Fixes: #145092

Author: DarrylWong

pkg/cmd/roachtest/roachtestutil/utils.go

@@ -361,6 +361,29 @@ func ExecWithRetry(
 	return result, err
 }
 
+// QueryWithRetry queries the given SQL statement with the specified retry logic.
+func QueryWithRetry(
+	ctx context.Context,
+	l *logger.Logger,
+	db *gosql.DB,
+	retryOpts retry.Options,
+	query string,
+	args ...any,
+) (*gosql.Rows, error) {
+	var rows *gosql.Rows
+	err := retryOpts.Do(ctx, func(ctx context.Context) error {
+		var err error
+		rows, err = db.QueryContext(ctx, query, args...)
+		if err != nil {
+			l.Printf("%s failed (retrying): %v", query, err)
+			return err
+		}
+		return nil
+	})
+
+	return rows, err
+}
+
 // ClusterSettingRetryOpts are retry options intended for cluster setting operations.
 //
 // We use relatively high backoff parameters with the assumption that:

pkg/cmd/roachtest/roachtestutil/validation_check.go

@@ -103,14 +103,18 @@ FROM crdb_internal.check_consistency(false, '', '') as t;`)
 
 // CheckInvalidDescriptors returns an error if there exists any descriptors in
 // the crdb_internal.invalid_objects virtual table.
-func CheckInvalidDescriptors(ctx context.Context, db *gosql.DB) error {
+func CheckInvalidDescriptors(ctx context.Context, l *logger.Logger, db *gosql.DB) error {
 	var invalidIDs string
+	l.Printf("checking for invalid descriptors")
 	if err := timeutil.RunWithTimeout(ctx, "descriptor validation", time.Minute, func(ctx context.Context) error {
 		// Because crdb_internal.invalid_objects is a virtual table, by default, the
 		// query will take a lease on the database sqlDB is connected to and only run
 		// the query on the given database. The "" prefix prevents this lease
 		// acquisition and allows the query to fetch all descriptors in the cluster.
-		rows, err := db.QueryContext(ctx, `SELECT id, obj_name, error FROM "".crdb_internal.invalid_objects`)
+		//
+		// This often times out when the cluster is overloaded. Since this is just
+		// for validation, we retry with a relatively high backoff.
+		rows, err := QueryWithRetry(ctx, l, db, ClusterSettingRetryOpts, `SELECT id, obj_name, error FROM "".crdb_internal.invalid_objects`)
 		if err != nil {
 			return err
 		}

pkg/cmd/roachtest/test_runner.go

@@ -1638,7 +1638,7 @@ func (r *testRunner) postTestAssertions(
 				// NB: the invalid description checks should run at the system tenant level.
 				db := c.Conn(ctx, t.L(), validationNode, option.VirtualClusterName(install.SystemInterfaceName))
 				defer db.Close()
-				if err := roachtestutil.CheckInvalidDescriptors(ctx, db); err != nil {
+				if err := roachtestutil.CheckInvalidDescriptors(ctx, t.L(), db); err != nil {
 					postAssertionErr(errors.WithDetail(err, "invalid descriptors check failed"))
 				}
 			}()

pkg/cmd/roachtest/tests/mixed_version_backup.go

@@ -1289,7 +1289,7 @@ func (d *BackupRestoreTestDriver) verifyBackupCollection(
 		}
 	}
 	_, db := d.testUtils.RandomDB(rng, d.testUtils.roachNodes)
-	if err := roachtestutil.CheckInvalidDescriptors(ctx, db); err != nil {
+	if err := roachtestutil.CheckInvalidDescriptors(ctx, l, db); err != nil {
 		return fmt.Errorf("failed descriptor check: %w", err)
 	}