Merge #143399

143399: roachtest: "large" cluster tweaks r=herkolategan,darrylwong a=srosenberg

While triaging recent failures of the
_weekly_ `multi-region/mixed-version`,
we observed that the 1h upgrade timeout
was exceeded, potentially due to [1].
We also observed that 10m wasn't sufficient
to construct `debug.zip`.

Thus, we bump the upgrade timeout to 2h and
`FetchDebugZip` to 20m. We also reduce the
cluster size from 80 nodes down to 52, in
hope of stabilizing it. Finally, we skip
`FetchTimeSeries` since we have no direct
use for it, and it has non-negligible overhead.

[1] #141420

Informs: #121455
Epic: none
Release note: None

Co-authored-by: Stan Rosenberg <[email protected]>

Author: craig[bot]

pkg/cmd/roachtest/cluster.go

@@ -1395,9 +1395,16 @@ func (c *clusterImpl) FetchDebugZip(
 	c.status("fetching debug zip")
 
 	nodes := selectedNodesOrDefault(opts, c.All())
+	// Shuffle the nodes to avoid always trying the same node first.
+	rand.Shuffle(len(nodes), func(i, j int) { nodes[i], nodes[j] = nodes[j], nodes[i] })
+	defaultTimeout := 10 * time.Minute
+	if c.spec.NodeCount >= 30 {
+		// For "large" clusters, double the timeout.
+		defaultTimeout *= 2
+	}
 
 	// Don't hang forever if we can't fetch the debug zip.
-	return timeutil.RunWithTimeout(ctx, "debug zip", 10*time.Minute, func(ctx context.Context) error {
+	return timeutil.RunWithTimeout(ctx, "debug zip", defaultTimeout, func(ctx context.Context) error {
 		const zipName = "debug.zip"
 		path := filepath.Join(c.t.ArtifactsDir(), dest)
 		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {

pkg/cmd/roachtest/test_runner.go

@@ -1753,8 +1753,11 @@ func (r *testRunner) collectArtifacts(
 		if err := c.FetchPebbleCheckpoints(ctx, t.L()); err != nil {
 			t.L().Printf("failed to fetch Pebble checkpoints: %s", err)
 		}
-		if err := c.FetchTimeseriesData(ctx, t.L()); err != nil {
-			t.L().Printf("failed to fetch timeseries data: %s", err)
+		// Bypass the collection of timeseries data for "large" clusters.
+		if c.spec.NodeCount < 30 {
+			if err := c.FetchTimeseriesData(ctx, t.L()); err != nil {
+				t.L().Printf("failed to fetch timeseries data: %s", err)
+			}
 		}
 		if err := c.FetchDebugZip(ctx, t.L(), "debug.zip"); err != nil {
 			t.L().Printf("failed to collect zip: %s", err)

pkg/cmd/roachtest/tests/mixed_version_multi_region.go

@@ -36,7 +36,7 @@ func registerMultiRegionMixedVersion(r registry.Registry) {
 	}
 
 	const (
-		nodesPerRegion = 20
+		nodesPerRegion = 13
 		// These values are somewhat arbitrary: currently, they are
 		// sufficient to keep the cluster relatively busy (CPU utilization
 		// varying from 10-60%). In the future, these values might be
@@ -75,7 +75,7 @@ func registerMultiRegionMixedVersion(r registry.Registry) {
 				mixedversion.NeverUseFixtures,
 				// Allow migrations to run for a longer period of time due to
 				// added latency and cluster size.
-				mixedversion.UpgradeTimeout(1*time.Hour),
+				mixedversion.UpgradeTimeout(2*time.Hour),
 				// There are known issues upgrading from older patch releases
 				// in MR clusters (e.g., #113908), so use the latest patch
 				// releases to avoid flakes.