Merge #149283

149283: roachtest: deflake splits/largerange/size=32GiB r=iskettaneh a=iskettaneh

This commit reduces the snapshot_rebalance.max_rate from 512 to 192. The reasons are: (1) Even on Azure and GCE, I saw that we get about 192 Mbps of an actual snapshot send rate. (2) On IBM, the rate is much lower. (3) This rate goes into calculating the snapshot sending timeout, and it should generally be lower or slightly higher than the actual snapshot sending rate. If it's 10 times lower than the actual rate, the snapshot will most likely timeout.

See this comment for the exact analysis:
#148982 (comment)

Fixes: #148982

Release note: None

Co-authored-by: iskettaneh <[email protected]>

Author: craig[bot]

pkg/cmd/roachtest/tests/split.go

@@ -644,7 +644,12 @@ func runLargeRangeSplits(ctx context.Context, t test.Test, c cluster.Cluster, si
 			if _, err := db.ExecContext(ctx, fmt.Sprintf("SET CLUSTER SETTING kv.range.range_size_hard_cap = '%d'", rangeMaxSize*2)); err != nil {
 				return err
 			}
-			if _, err := db.ExecContext(ctx, `SET CLUSTER SETTING kv.snapshot_rebalance.max_rate='512MiB'`); err != nil {
+
+			// Setting the max snapshot rebalance rate to a very high value like
+			// 512MiB could cause the snapshot-copy operation to timeout if the actual
+			// copy rate is significantly lower than  that. See #148982 for more
+			// details.
+			if _, err := db.ExecContext(ctx, `SET CLUSTER SETTING kv.snapshot_rebalance.max_rate='192MiB'`); err != nil {
 				return err
 			}
 			// This test splits an exceptionally large range. Disable MVCC stats