roachtest: use versioned bank workloads for mixed version

As of #149374, the bank workload is no longer backwards compatible.
This changes mixed version tests that use the bank workload
to upload a versioned binary and use that to import the workload.

Author: DarrylWong

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go

@@ -826,8 +826,16 @@ func (t *Test) BackgroundCommand(
 // if passed, is the command run to initialize the workload; it is run
 // synchronously as a regular startup function. `runCmd` is the
 // command to actually run the command; it is run in the background.
+//
+// If overrideBinary is true, the binary used to run the command(s) will
+// be replaced with the cockroach binary of the current version the
+// cluster is running in.
+// TODO(testeng): Replace with https://github.com/cockroachdb/cockroach/issues/147374
 func (t *Test) Workload(
-	name string, node option.NodeListOption, initCmd, runCmd *roachtestutil.Command,
+	name string,
+	node option.NodeListOption,
+	initCmd, runCmd *roachtestutil.Command,
+	overrideBinary bool,
 ) StopFunc {
 	seed := uint64(t.prng.Int63())
 	addSeed := func(cmd *roachtestutil.Command) {
@@ -838,11 +846,31 @@ func (t *Test) Workload(
 
 	if initCmd != nil {
 		addSeed(initCmd)
-		t.OnStartup(fmt.Sprintf("initialize %s workload", name), t.runCommandFunc(node, initCmd.String()))
+		t.OnStartup(fmt.Sprintf("initialize %s workload", name), func(ctx context.Context, l *logger.Logger, rng *rand.Rand, h *Helper) error {
+			if overrideBinary {
+				binary, err := clusterupgrade.UploadCockroach(ctx, t.rt, t.logger, t.cluster, node, h.System.FromVersion)
+				if err != nil {
+					t.rt.Fatal(err)
+				}
+				initCmd.Binary = binary
+			}
+			l.Printf("running command `%s` on nodes %v", initCmd.String(), node)
+			return t.cluster.RunE(ctx, option.WithNodes(node), initCmd.String())
+		})
 	}
 
 	addSeed(runCmd)
-	return t.BackgroundCommand(fmt.Sprintf("%s workload", name), node, runCmd)
+	return t.BackgroundFunc(fmt.Sprintf("%s workload", name), func(ctx context.Context, l *logger.Logger, rng *rand.Rand, h *Helper) error {
+		if overrideBinary {
+			binary, err := clusterupgrade.UploadCockroach(ctx, t.rt, t.logger, t.cluster, node, h.System.FromVersion)
+			if err != nil {
+				t.rt.Fatal(err)
+			}
+			runCmd.Binary = binary
+		}
+		l.Printf("running command `%s` on nodes %v", runCmd.String(), node)
+		return t.cluster.RunE(ctx, option.WithNodes(node), runCmd.String())
+	})
 }
 
 // Run is like RunE, except it fatals the test if any error occurs.

pkg/cmd/roachtest/roachtestutil/mixedversion/planner_test.go

@@ -126,7 +126,7 @@ func TestTestPlanner(t *testing.T) {
 			case "workload":
 				initCmd := roachtestutil.NewCommand("./cockroach workload init some-workload")
 				runCmd := roachtestutil.NewCommand("./cockroach workload run some-workload")
-				mvt.Workload(d.CmdArgs[0].Vals[0], nodes, initCmd, runCmd)
+				mvt.Workload(d.CmdArgs[0].Vals[0], nodes, initCmd, runCmd, false /* overrideBinary */)
 			case "background-command":
 				cmd := roachtestutil.NewCommand("./cockroach some-command")
 				mvt.BackgroundCommand(d.CmdArgs[0].Vals[0], nodes, cmd)

pkg/cmd/roachtest/tests/mixed_version_backup.go

@@ -2954,8 +2954,13 @@ func registerBackupMixedVersion(r registry.Registry) {
 			//   the cluster relatively busy while the backup and restores
 			//   take place. Its schema is also more complex, and the
 			//   operations more closely resemble a customer workload.
-			stopBank := mvt.Workload("bank", c.WorkloadNode(), bankInit, bankRun)
-			stopTPCC := mvt.Workload("tpcc", c.WorkloadNode(), tpccInit, tpccRun)
+
+			// Use the same versioned workload binary as the cluster. The bank workload
+			// is no longer backwards compatible after #149374, so we need to use the same
+			// version as the cockroach cluster.
+			// TODO(testeng): Replace with https://github.com/cockroachdb/cockroach/issues/147374
+			stopBank := mvt.Workload("bank", c.WorkloadNode(), bankInit, bankRun, true /* overrideBinary */)
+			stopTPCC := mvt.Workload("tpcc", c.WorkloadNode(), tpccInit, tpccRun, false /* overrideBinary */)
 			stopSystemWriter := mvt.BackgroundFunc("system table writer", backupTest.systemTableWriter)
 
 			mvt.InMixedVersion("plan and run backups", backupTest.planAndRunBackups)

pkg/cmd/roachtest/tests/mixed_version_c2c.go

@@ -276,7 +276,7 @@ func (cm *c2cMixed) WorkloadHook(ctx context.Context) {
 		Arg("{pgurl%s}", cm.c.Range(1, cm.sp.srcNodes)).
 		Option("tolerate-errors").
 		Flag("warehouses", 500)
-	cm.workloadStopper = cm.sourceMvt.Workload("tpcc", cm.c.WorkloadNode(), tpccInitCmd, tpccRunCmd)
+	cm.workloadStopper = cm.sourceMvt.Workload("tpcc", cm.c.WorkloadNode(), tpccInitCmd, tpccRunCmd, false /* overrideBinary */)
 
 	readerTenantName := fmt.Sprintf("%s-readonly", destTenantName)
 
@@ -286,7 +286,7 @@ func (cm *c2cMixed) WorkloadHook(ctx context.Context) {
 		Flag("warehouses", 500).
 		Flag("mix", "newOrder=0,payment=0,orderStatus=1,delivery=0,stockLevel=1")
 
-	cm.readOnlyWorkloadStopper = cm.destMvt.Workload("tpcc-read-only", cm.c.WorkloadNode(), nil, tpccStandbyRunCmd)
+	cm.readOnlyWorkloadStopper = cm.destMvt.Workload("tpcc-read-only", cm.c.WorkloadNode(), nil, tpccStandbyRunCmd, false /* overrideBinary */)
 
 }
 

pkg/cmd/roachtest/tests/mixed_version_ldr.go

@@ -258,10 +258,10 @@ func (lm *ldrMixed) SetupHook(ctx context.Context) {
 
 func (lm *ldrMixed) WorkloadHook(ctx context.Context) {
 	leftWorkloadCmd := workloadRunCmd(lm.sp.LeftNodesList())
-	lm.leftWorkloadStopper = lm.leftMvt.Workload("kv", lm.c.WorkloadNode(), nil, leftWorkloadCmd)
+	lm.leftWorkloadStopper = lm.leftMvt.Workload("kv", lm.c.WorkloadNode(), nil, leftWorkloadCmd, false /* overrideBinary */)
 
 	rightWorkloadCmd := workloadRunCmd(lm.sp.RightNodesList())
-	lm.rightWorkloadStopper = lm.rightMvt.Workload("kv", lm.c.WorkloadNode(), nil, rightWorkloadCmd)
+	lm.rightWorkloadStopper = lm.rightMvt.Workload("kv", lm.c.WorkloadNode(), nil, rightWorkloadCmd, false /* overrideBinary */)
 }
 
 func (lm *ldrMixed) LatencyHook(ctx context.Context) {

pkg/cmd/roachtest/tests/mixed_version_sql_stats.go

@@ -74,7 +74,7 @@ func runSQLStatsMixedVersion(ctx context.Context, t test.Test, c cluster.Cluster
 	}
 
 	mvt.OnStartup("set cluster settings", setClusterSettings)
-	stopTpcc := mvt.Workload("tpcc", c.WorkloadNode(), initWorkload, runWorkload)
+	stopTpcc := mvt.Workload("tpcc", c.WorkloadNode(), initWorkload, runWorkload, false /* overrideBinary */)
 	defer stopTpcc()
 
 	requestTypes := map[string]serverpb.CombinedStatementsStatsRequest_StatsType{

pkg/cmd/roachtest/tests/tpcc.go

@@ -602,11 +602,16 @@ func runTPCCMixedHeadroom(ctx context.Context, t test.Test, c cluster.Cluster) {
 	// upgrade machinery, in which a) all ranges are touched and b) work proportional
 	// to the amount data may be carried out.
 	importLargeBank := func(ctx context.Context, l *logger.Logger, rng *rand.Rand, h *mixedversion.Helper) error {
+		randomNode := c.Node(c.CRDBNodes().SeededRandNode(rng)[0])
+		// Upload a versioned cockroach binary to the random node. The bank workload
+		// is no longer backwards compatible after #149374, so we need to use the same
+		// version as the cockroach cluster.
+		// TODO(testeng): Replace with https://github.com/cockroachdb/cockroach/issues/147374
+		binary := uploadCockroach(ctx, t, c, randomNode, h.System.FromVersion)
 		l.Printf("waiting for tenant features to be enabled")
 		<-tenantFeaturesEnabled
 
-		randomNode := c.Node(c.CRDBNodes().SeededRandNode(rng)[0])
-		cmd := roachtestutil.NewCommand("%s workload fixtures import bank", test.DefaultCockroachPath).
+		cmd := roachtestutil.NewCommand("%s workload fixtures import bank", binary).
 			Arg("{pgurl%s}", randomNode).
 			Flag("payload-bytes", 10240).
 			Flag("rows", bankRows).