Merge #155106

155106: mixedversion: disable upreplication mutator r=srosenberg a=DarrylWong

This mutator can cause timeouts upreplicating non trivial amounts of data. It should be default disabled except for curated tests.

Informs: #142194
Fixes: #151984
Fixes: #154838
Release note: none
Epic: none

Co-authored-by: DarrylWong <[email protected]>

Author: craig[bot]

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go

@@ -355,6 +355,7 @@ type (
 		overriddenMutatorProbabilities map[string]float64
 		hooksSupportFailureInjection   bool
 		workloadNodes                  option.NodeListOption
+		enableUpReplication            bool
 	}
 
 	CustomOption func(*testOptions)
@@ -413,6 +414,12 @@ func EnableHooksDuringFailureInjection(opts *testOptions) {
 	opts.hooksSupportFailureInjection = true
 }
 
+// EnableUpReplication is an option that can be passed to `NewTest` to enable
+// up-replication to 5X before panicking a node during failure injection tests.
+func EnableUpReplication(opts *testOptions) {
+	opts.enableUpReplication = true
+}
+
 // NeverUseFixtures is an option that can be passed to `NewTest` to
 // disable the use of fixtures in the test. Necessary if the test
 // wants to use a number of cockroach nodes other than 4.

pkg/cmd/roachtest/roachtestutil/mixedversion/mutators.go

@@ -411,7 +411,7 @@ func (m panicNodeMutator) Generate(
 
 	// If we have at least 5 nodes, we can safely upreplicate to 5X before panicking a node.
 	// This allows for a longer panic duration before recovery.
-	supportsUpReplication := len(nodeList) >= 5
+	supportsUpReplication := planner.options.enableUpReplication && len(nodeList) >= 5
 
 	for _, upgrade := range upgrades {
 		possiblePointsInTime := upgrade.

pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/separate_process

@@ -44,8 +44,8 @@ Plan:
 ├── delete all-tenants override for the `version` key (9) [stage=system:tenant-setup;tenant:tenant-setup]
 ├── run startup hooks concurrently
 │   ├── run "create tables" hookId="planner_test.go:141", after 3m0s delay (10) [stage=system:on-startup;tenant:on-startup]
-│   └── run "initialize bank workload" hookId="mixedversion.go:877", after 100ms delay (11) [stage=system:on-startup;tenant:on-startup]
-├── run "bank workload" hookId="mixedversion.go:885" (12) [stage=system:background;tenant:background]
+│   └── run "initialize bank workload" hookId="mixedversion.go:884", after 100ms delay (11) [stage=system:on-startup;tenant:on-startup]
+├── run "bank workload" hookId="mixedversion.go:892" (12) [stage=system:background;tenant:background]
 ├── upgrade cluster from "v22.2.3" to "v23.1.4"
 │   ├── upgrade storage cluster
 │   │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (13) [stage=system:init;tenant:upgrading-system]

pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/shared_process

@@ -53,8 +53,8 @@ Plan:
 ├── delete all-tenants override for the `version` key (16) [stage=system:tenant-setup;tenant:tenant-setup]
 ├── run startup hooks concurrently
 │   ├── run "create tables" hookId="planner_test.go:141", after 30s delay (17) [stage=system:on-startup;tenant:on-startup]
-│   └── run "initialize bank workload" hookId="mixedversion.go:877", after 3m0s delay (18) [stage=system:on-startup;tenant:on-startup]
-├── run "bank workload" hookId="mixedversion.go:885" (19) [stage=system:background;tenant:background]
+│   └── run "initialize bank workload" hookId="mixedversion.go:884", after 3m0s delay (18) [stage=system:on-startup;tenant:on-startup]
+├── run "bank workload" hookId="mixedversion.go:892" (19) [stage=system:background;tenant:background]
 ├── upgrade cluster from "v23.1.12" to "v23.2.0"
 │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (20) [stage=system:init;tenant:init]
 │   ├── prevent auto-upgrades on mixed-version-tenant-cyvju tenant by setting `preserve_downgrade_option` (21) [stage=system:init;tenant:init]

pkg/cmd/roachtest/roachtestutil/mixedversion/testdata/planner/step_stages

@@ -35,10 +35,10 @@ Plan:
 ├── install fixtures for version "v22.2.3" (1) [stage=system-setup]
 ├── start cluster at version "v22.2.3" (2) [stage=system-setup]
 ├── wait for all nodes (:1-4) to acknowledge cluster version '22.2' on system tenant (3) [stage=system-setup]
-├── run "initialize bank workload" hookId="mixedversion.go:877" (4) [stage=on-startup]
+├── run "initialize bank workload" hookId="mixedversion.go:884" (4) [stage=on-startup]
 ├── start background hooks concurrently
-│   ├── run "bank workload" hookId="mixedversion.go:885", after 0s delay (5) [stage=background]
-│   └── run "csv server" hookId="mixedversion.go:854", after 0s delay (6) [stage=background]
+│   ├── run "bank workload" hookId="mixedversion.go:892", after 0s delay (5) [stage=background]
+│   └── run "csv server" hookId="mixedversion.go:861", after 0s delay (6) [stage=background]
 ├── upgrade cluster from "v22.2.3" to "v23.1.4"
 │   ├── prevent auto-upgrades on system tenant by setting `preserve_downgrade_option` (7) [stage=init]
 │   ├── upgrade nodes :1-4 from "v22.2.3" to "v23.1.4"

pkg/cmd/roachtest/tests/tpcc.go

@@ -600,6 +600,7 @@ func runTPCCMixedHeadroom(ctx context.Context, t test.Test, c cluster.Cluster, c
 		customOpts = append([]mixedversion.CustomOption{
 			mixedversion.NeverUseFixtures,
 			mixedversion.EnableHooksDuringFailureInjection,
+			mixedversion.EnableUpReplication,
 		},
 			customOpts...)
 	}