roachtest: add fingerprint validation to classic restore roachtests

With the switch to the new fixtures in #146555, we temporarily removed
fingerprint validation. This commit teaches the restore roachtests to
fingerprint after restore, validating the results against the
fingerprints stored in the fixtures (introduced in #147662.

Epic: CRDB-50823

Fixes: #148962

Release note: None

Author: kev-cao

pkg/cmd/roachtest/tests/backup_fixtures.go

@@ -7,6 +7,7 @@ package tests
 
 import (
 	"context"
+	gosql "database/sql"
 	"fmt"
 	"net/url"
 	"path"
@@ -25,6 +26,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachprod"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/blobfixture"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/vm"
 	"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
@@ -412,20 +414,52 @@ func (bd *backupDriver) queryJobStates(
 func (bd *backupDriver) fingerprintFixture(ctx context.Context) map[string]string {
 	conn := bd.c.Conn(ctx, bd.t.L(), 1)
 	defer conn.Close()
-	sql := sqlutils.MakeSQLRunner(conn)
-	aost := bd.getLatestAOST(ctx, sql)
-	tables := getDatabaseTables(ctx, bd.t, sql, bd.sp.fixture.DatabaseName())
+	return fingerprintDatabase(
+		ctx, bd.t, conn,
+		bd.sp.fixture.DatabaseName(), bd.getLatestAOST(sqlutils.MakeSQLRunner(conn)),
+	)
+}
 
-	m := bd.c.NewDeprecatedMonitor(ctx)
+// getLatestAOST returns the end time as seen in SHOW BACKUP of the latest
+// backup in the fixture.
+func (bd *backupDriver) getLatestAOST(sql *sqlutils.SQLRunner) string {
+	uri := bd.registry.URI(bd.fixture.DataPath)
+	query := fmt.Sprintf(
+		`SELECT end_time FROM
+		[SHOW BACKUP FROM LATEST IN '%s']
+		ORDER BY end_time DESC
+		LIMIT 1`,
+		uri.String(),
+	)
+	var endTime string
+	sql.QueryRow(bd.t, query).Scan(&endTime)
+	return endTime
+}
+
+// fingerprintDatabase fingerprints all of the tables in the provided database
+// and returns a map of fully qualified table names to their fingerprints.
+// If AOST is not provided, the current time is used as the AOST.
+func fingerprintDatabase(
+	ctx context.Context, t test.Test, conn *gosql.DB, dbName string, aost string,
+) map[string]string {
+	sql := sqlutils.MakeSQLRunner(conn)
+	tables := getDatabaseTables(ctx, t, sql, dbName)
+	if len(tables) == 0 {
+		t.L().Printf("no tables found in database %s", dbName)
+		return nil
+	}
+	t.L().Printf("fingerprinting %d tables in database %s", len(tables), dbName)
 
-	bd.t.L().Printf("fingerprinting %d tables in %s", len(tables), bd.sp.fixture.DatabaseName())
 	fingerprints := make(map[string]string)
 	var mu syncutil.Mutex
 	start := timeutil.Now()
+	group := t.NewErrorGroup()
 	for _, table := range tables {
-		m.Go(func(ctx context.Context) error {
+		group.Go(func(ctx context.Context, log *logger.Logger) error {
 			fpContents := newFingerprintContents(conn, table)
-			if err := fpContents.Load(ctx, bd.t.L(), aost, nil /* tableContents */); err != nil {
+			if err := fpContents.Load(
+				ctx, log, aost, nil, /* tableContents */
+			); err != nil {
 				return err
 			}
 			mu.Lock()
@@ -434,31 +468,14 @@ func (bd *backupDriver) fingerprintFixture(ctx context.Context) map[string]strin
 			return nil
 		})
 	}
-	m.Wait()
-	bd.t.L().Printf(
+	require.NoError(t, group.WaitE(), "error fingerprinting tables in database %s", dbName)
+	t.L().Printf(
 		"fingerprinted %d tables in %s in %s",
-		len(tables), bd.sp.fixture.DatabaseName(), timeutil.Since(start),
+		len(tables), dbName, timeutil.Since(start),
 	)
-
 	return fingerprints
 }
 
-// getLatestAOST returns the end time as seen in SHOW BACKUP of the latest
-// backup in the fixture.
-func (bd *backupDriver) getLatestAOST(ctx context.Context, sql *sqlutils.SQLRunner) string {
-	uri := bd.registry.URI(bd.fixture.DataPath)
-	query := fmt.Sprintf(
-		`SELECT end_time FROM
-		[SHOW BACKUP FROM LATEST IN '%s']
-		ORDER BY end_time DESC
-		LIMIT 1`,
-		uri.String(),
-	)
-	var endTime string
-	sql.QueryRow(bd.t, query).Scan(&endTime)
-	return endTime
-}
-
 // getDatabaseTables returns the fully qualified name of every table in the
 // fixture.
 // Note: This assumes there aren't any funky characters in the identifiers, so

pkg/cmd/roachtest/tests/mixed_version_backup.go

@@ -622,8 +622,9 @@ func newFingerprintContents(db *gosql.DB, table string) *fingerprintContents {
 	return &fingerprintContents{db: db, table: table}
 }
 
-// Load computes the fingerprints for the underlying table and stores
-// the contents in the `fingeprints` field.
+// Load computes the fingerprints for the underlying table and stores the
+// contents in the `fingeprints` field. If timestamp is not set, computes
+// the fingerprint for the current time.
 func (fc *fingerprintContents) Load(
 	ctx context.Context, l *logger.Logger, timestamp string, _ tableContents,
 ) error {

pkg/cmd/roachtest/tests/online_restore.go

@@ -292,7 +292,7 @@ func registerOnlineRestorePerf(r registry.Registry) {
 						// Takes 10 minutes on OR tests for some reason.
 						SkipPostValidations: registry.PostValidationReplicaDivergence,
 						Run: func(ctx context.Context, t test.Test, c cluster.Cluster) {
-							rd := makeRestoreDriver(t, c, sp.restoreSpecs)
+							rd := makeRestoreDriver(ctx, t, c, sp.restoreSpecs)
 							rd.prepareCluster(ctx)
 
 							restoreStats := runRestore(
@@ -383,7 +383,7 @@ func registerOnlineRestoreCorrectness(r registry.Registry) {
 					t, sp, defaultSeed, defaultFakeTime, "-online.trace",
 				)
 
-				rd := makeRestoreDriver(t, c, sp.restoreSpecs)
+				rd := makeRestoreDriver(ctx, t, c, sp.restoreSpecs)
 				rd.prepareCluster(ctx)
 
 				runRestore(ctx, t, c, regRestoreSpecs, rd, false /* runOnline */, true /* runWorkload */)