Merge #153615

153615: roachtest: use set seed in backup/restore workloads r=msbutler a=kev-cao

An issue we face today with test failures on our backup/restore roachtests is reproducibility, especially in failures relating to corruption. Part of that reason is that when running our workloads in our roundtrip tests, we do not set a seed for the workloads. This patch updates our backup/restore roachtests to match the workload's seed to the test run's seed.

Epic: None

Release note: None

Co-authored-by: Kevin Cao <[email protected]>

Author: craig[bot]

pkg/cmd/roachtest/tests/BUILD.bazel

@@ -241,6 +241,7 @@ go_library(
         "//pkg/cmd/roachtest/grafana",
         "//pkg/cmd/roachtest/option",
         "//pkg/cmd/roachtest/registry",
+        "//pkg/cmd/roachtest/roachtestflags",
         "//pkg/cmd/roachtest/roachtestutil",
         "//pkg/cmd/roachtest/roachtestutil/clusterupgrade",
         "//pkg/cmd/roachtest/roachtestutil/mixedversion",

pkg/cmd/roachtest/tests/backup_restore_roundtrip.go

@@ -157,7 +157,7 @@ func backupRestoreRoundTrip(
 
 		dbs := []string{"bank", "tpcc", schemaChangeDB}
 		d, runBackgroundWorkload, _, err := createDriversForBackupRestore(
-			ctx, t, c, m, testRNG, testUtils, dbs,
+			ctx, t, c, m, testRNG, seed, testUtils, dbs,
 		)
 		if err != nil {
 			return err
@@ -243,6 +243,7 @@ func startBackgroundWorkloads(
 	c cluster.Cluster,
 	m cluster.Monitor,
 	testRNG *rand.Rand,
+	seed int64,
 	roachNodes, workloadNode option.NodeListOption,
 	testUtils *CommonTestUtils,
 	dbs []string,
@@ -254,9 +255,9 @@ func startBackgroundWorkloads(
 	if testUtils.mock {
 		numWarehouses = 10
 	}
-	tpccInit, tpccRun := tpccWorkloadCmd(l, testRNG, numWarehouses, roachNodes)
-	bankInit, bankRun := bankWorkloadCmd(l, testRNG, roachNodes, testUtils.mock)
-	scInit, scRun := schemaChangeWorkloadCmd(l, testRNG, roachNodes, testUtils.mock)
+	tpccInit, tpccRun := tpccWorkloadCmd(l, testRNG, seed, numWarehouses, roachNodes)
+	bankInit, bankRun := bankWorkloadCmd(l, testRNG, seed, roachNodes, testUtils.mock)
+	scInit, scRun := schemaChangeWorkloadCmd(l, testRNG, seed, roachNodes, testUtils.mock)
 	if err := prepSchemaChangeWorkload(ctx, workloadNode, testUtils, testRNG); err != nil {
 		return nil, err
 	}
@@ -420,11 +421,12 @@ func createDriversForBackupRestore(
 	c cluster.Cluster,
 	m cluster.Monitor,
 	rng *rand.Rand,
+	seed int64,
 	testUtils *CommonTestUtils,
 	dbs []string,
 ) (*BackupRestoreTestDriver, func() (func(), error), [][]string, error) {
 	runBackgroundWorkload, err := startBackgroundWorkloads(
-		ctx, t.L(), c, m, rng, c.CRDBNodes(), c.WorkloadNode(), testUtils, dbs,
+		ctx, t.L(), c, m, rng, seed, c.CRDBNodes(), c.WorkloadNode(), testUtils, dbs,
 	)
 	if err != nil {
 		return nil, nil, nil, err
@@ -460,7 +462,7 @@ func testOnlineRestoreRecovery(ctx context.Context, t test.Test, c cluster.Clust
 
 		dbs := []string{"bank", "tpcc", schemaChangeDB}
 		d, runBackgroundWorkload, _, err := createDriversForBackupRestore(
-			ctx, t, c, m, testRNG, testUtils, dbs,
+			ctx, t, c, m, testRNG, seed, testUtils, dbs,
 		)
 		if err != nil {
 			return err

pkg/cmd/roachtest/tests/mixed_version_backup.go

@@ -30,6 +30,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestflags"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil/clusterupgrade"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/roachtestutil/mixedversion"
@@ -2950,8 +2951,8 @@ func registerBackupMixedVersion(r registry.Registry) {
 			// for the cluster used in this test without overloading it,
 			// which can make the backups take much longer to finish.
 			const numWarehouses = 100
-			bankInit, bankRun := bankWorkloadCmd(t.L(), testRNG, c.CRDBNodes(), false)
-			tpccInit, tpccRun := tpccWorkloadCmd(t.L(), testRNG, numWarehouses, c.CRDBNodes())
+			bankInit, bankRun := bankWorkloadCmd(t.L(), testRNG, roachtestflags.GlobalSeed, c.CRDBNodes(), false)
+			tpccInit, tpccRun := tpccWorkloadCmd(t.L(), testRNG, roachtestflags.GlobalSeed, numWarehouses, c.CRDBNodes())
 
 			mvt.OnStartup("set short job interval", backupTest.setShortJobIntervals)
 			mvt.OnStartup("take backup in previous version", backupTest.maybeTakePreviousVersionBackup)
@@ -2989,23 +2990,29 @@ func registerBackupMixedVersion(r registry.Registry) {
 }
 
 func tpccWorkloadCmd(
-	l *logger.Logger, testRNG *rand.Rand, numWarehouses int, roachNodes option.NodeListOption,
+	l *logger.Logger,
+	testRNG *rand.Rand,
+	seed int64,
+	numWarehouses int,
+	roachNodes option.NodeListOption,
 ) (init *roachtestutil.Command, run *roachtestutil.Command) {
 	init = roachtestutil.NewCommand("./cockroach workload init tpcc").
 		MaybeOption(testRNG.Intn(2) == 0, "families").
 		Arg("{pgurl%s}", roachNodes).
-		Flag("warehouses", numWarehouses)
+		Flag("warehouses", numWarehouses).
+		MaybeFlag(seed != 0, "seed", seed)
 	run = roachtestutil.NewCommand("./cockroach workload run tpcc").
 		Arg("{pgurl%s}", roachNodes).
 		Flag("warehouses", numWarehouses).
+		MaybeFlag(seed != 0, "seed", seed).
 		Option("tolerate-errors")
 	l.Printf("tpcc init: %s", init)
 	l.Printf("tpcc run: %s", run)
 	return init, run
 }
 
 func bankWorkloadCmd(
-	l *logger.Logger, testRNG *rand.Rand, roachNodes option.NodeListOption, mock bool,
+	l *logger.Logger, testRNG *rand.Rand, seed int64, roachNodes option.NodeListOption, mock bool,
 ) (init *roachtestutil.Command, run *roachtestutil.Command) {
 	bankRows := bankPossibleRows[testRNG.Intn(len(bankPossibleRows))]
 	possiblePayloads := bankPossiblePayloadBytes
@@ -3024,31 +3031,36 @@ func bankWorkloadCmd(
 	init = roachtestutil.NewCommand("./cockroach workload init bank").
 		Flag("rows", bankRows).
 		MaybeFlag(bankPayload != 0, "payload-bytes", bankPayload).
+		MaybeFlag(seed != 0, "seed", seed).
 		Flag("ranges", 0).
 		Arg("{pgurl%s}", roachNodes)
 	run = roachtestutil.NewCommand("./cockroach workload run bank").
 		Arg("{pgurl%s}", roachNodes).
+		MaybeFlag(seed != 0, "seed", seed).
 		Option("tolerate-errors")
 	l.Printf("bank init: %s", init)
 	l.Printf("bank run: %s", run)
 	return init, run
 }
 
 func schemaChangeWorkloadCmd(
-	l *logger.Logger, testRNG *rand.Rand, roachNodes option.NodeListOption, mock bool,
+	l *logger.Logger, testRNG *rand.Rand, seed int64, roachNodes option.NodeListOption, mock bool,
 ) (init *roachtestutil.Command, run *roachtestutil.Command) {
 	maxOps := 1000
 	concurrency := 5
 	if mock {
 		maxOps = 10
 		concurrency = 2
 	}
-	initCmd := roachtestutil.NewCommand("./workload init schemachange").
+	if seed == 0 {
+		seed = testRNG.Int63()
+	}
+	initCmd := roachtestutil.NewCommand("COCKROACH_RANDOM_SEED=%d ./workload init schemachange", seed).
 		Arg("{pgurl%s}", roachNodes)
 	// TODO (msbutler): ideally we'd use the `db` flag to explicitly set the
 	// database, but it is currently broken:
 	// https://github.com/cockroachdb/cockroach/issues/115545
-	runCmd := roachtestutil.NewCommand("COCKROACH_RANDOM_SEED=%d ./workload run schemachange", testRNG.Int63()).
+	runCmd := roachtestutil.NewCommand("COCKROACH_RANDOM_SEED=%d ./workload run schemachange", seed).
 		Flag("verbose", 1).
 		Flag("max-ops", maxOps).
 		Flag("concurrency", concurrency).