roachtest: unconditionally save clusters that show raft fatal errors

When a cluster's logs contain a raft panic, it will be extended (by a week),
volume snapshots will be taken, and the cluster will not be destroyed. This
gives us the artifacts for a thorough investigation.

Verified manually via:

```
run --local acceptance/invariant-check-detection/failed=true
```

Here is the (editorialized) output:

```
test-teardown: 2025/05/20 08:15:15 cluster.go:2559: running cmd `([ -d logs ] && grep -RE '^...` on nodes [:1-4]; details in run_081515.744363000_n1-4_d-logs-grep-RE-Fraft.log
test-teardown: 2025/05/20 08:15:16 cluster.go:2995: extending cluster by 168h0m0s
test-teardown: 2025/05/20 08:15:16 cluster.go:1104: saving cluster local [tag:] (4 nodes) for debugging (--debug specified)
test-teardown: 2025/05/20 08:15:16 test_impl.go:478: test failure #2: full stack retained in failure_2.log: (test_runner.go:1705).maybeSaveClusterDueToInvariantProblems: invariant problem - snap name invariant-problem-local-8897676895823393049:
logs/foo.log:F250502 11:37:20.387424 1036 raft/raft.go:2411 ⋮ [T1,Vsystem,n1,s1,r155/1:?/Table/113/1/{43/578…-51/201…}?] 80 match(30115) is out of range [lastIndex(30114)]. Was the raft log corrupted, truncated, or lost?
```

Closes #145953.
Informs #146617.
Informs #138028.

Epic: none

Author: tbg

pkg/cmd/roachtest/test_runner.go

@@ -1615,6 +1615,11 @@ func (r *testRunner) postTestAssertions(
 func (r *testRunner) teardownTest(
 	ctx context.Context, t *testImpl, c *clusterImpl, timedOut bool,
 ) error {
+	// Check for rare conditions (such as storage durability crashes) at this
+	// point. This may still mark the test as failed (so that we enter artifacts
+	// collection below).
+	r.maybeSaveClusterDueToInvariantProblems(ctx, t, c)
+
 	if timedOut || t.Failed() || roachtestflags.AlwaysCollectArtifacts {
 		err := r.collectArtifacts(ctx, t, c, timedOut, time.Hour)
 		if err != nil {
@@ -1669,6 +1674,43 @@ func (r *testRunner) teardownTest(
 	return nil
 }
 
+// maybeSaveClusterDueToInvariantProblems detects rare conditions (such as
+// storage durability crashes) on the cluster and if one is detected,
+// unconditionally preserves the cluster for future debugging. It also creates
+// volume snapshots so that the durable state close to the incident is
+// preserved.
+func (r *testRunner) maybeSaveClusterDueToInvariantProblems(
+	ctx context.Context, t *testImpl, c *clusterImpl,
+) {
+	if len(c.Nodes()) == 0 {
+		return // test only
+	}
+	dets, err := c.RunWithDetails(ctx, t.L(), option.WithNodes(c.All()),
+		"([ -d logs ] && grep -RE '^F.*raft' logs) || true",
+	)
+	for _, det := range dets {
+		err = errors.CombineErrors(err, det.Err)
+	}
+	if err != nil {
+		t.L().Printf(
+			"failed to check whether to save cluster due to invariant problems: %s",
+			err,
+		)
+		return
+	}
+
+	for _, det := range dets {
+		if det.Stdout != "" {
+			_ = c.Extend(ctx, 7*24*time.Hour, t.L())
+			snapName := "invariant-problem-" + c.Name() + "-" + strconv.Itoa(rand.Int())
+			_, _ = c.CreateSnapshot(ctx, snapName)
+			c.Save(ctx, "invariant problem - snap name "+snapName, t.L())
+			t.Error("invariant problem - snap name " + snapName + ":\n" + det.Stdout)
+			return
+		}
+	}
+}
+
 func (r *testRunner) collectArtifacts(
 	ctx context.Context, t *testImpl, c *clusterImpl, timedOut bool, timeout time.Duration,
 ) error {

pkg/cmd/roachtest/tests/acceptance.go

@@ -11,10 +11,13 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/cluster"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/option"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/registry"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/spec"
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
 	"github.com/cockroachdb/errors"
+	"github.com/stretchr/testify/require"
 )
 
 func registerAcceptance(r registry.Registry) {
@@ -81,6 +84,22 @@ func registerAcceptance(r registry.Registry) {
 				// and version upgrade is impossible to test as of 05/2025.
 				incompatibleClouds: registry.OnlyIBM,
 			},
+			// Tests for maybeSaveClusterDueToInvariantProblems. These don't verify
+			// that everything works as it should, but they can be run to verify
+			// manually that the cluster is saved correctly and the log output is
+			// helpful.
+			{
+				name:    "invariant-check-detection/failed=true",
+				fn:      runInvariantCheckDetectionFailed,
+				timeout: time.Hour,
+				skip:    "manual only",
+			},
+			{
+				name:    "invariant-check-detection/failed=false",
+				fn:      runInvariantCheckDetectionSuccess,
+				timeout: time.Hour,
+				skip:    "manual only",
+			},
 		},
 		registry.OwnerDisasterRecovery: {
 			{
@@ -145,7 +164,9 @@ func registerAcceptance(r registry.Registry) {
 					tc.name,
 				))
 			}
-			suites := append([]string{registry.Nightly, registry.Quick, registry.Acceptance}, tc.suites...)
+			var suites []string
+			suites = append(suites, registry.Nightly, registry.Quick, registry.Acceptance)
+			suites = append(suites, tc.suites...)
 			testSpec := registry.TestSpec{
 				Name:              "acceptance/" + tc.name,
 				Owner:             owner,
@@ -171,3 +192,24 @@ func registerAcceptance(r registry.Registry) {
 		}
 	}
 }
+
+func runInvariantCheckDetectionFailed(ctx context.Context, t test.Test, c cluster.Cluster) {
+	runInvariantCheckDetection(ctx, t, c, true)
+}
+
+func runInvariantCheckDetectionSuccess(ctx context.Context, t test.Test, c cluster.Cluster) {
+	runInvariantCheckDetection(ctx, t, c, false)
+}
+
+func runInvariantCheckDetection(ctx context.Context, t test.Test, c cluster.Cluster, failed bool) {
+	c.Put(ctx, t.Cockroach(), "cockroach")
+	c.Start(ctx, t.L(), option.DefaultStartOpts(), install.MakeClusterSettings(), c.Range(1, 3))
+	require.NoError(t, c.PutString(ctx, `
+foo br baz
+F250502 11:37:20.387424 1036 raft/raft.go:2411 ⋮ [T1,Vsystem,n1,s1,r155/1:‹/Table/113/1/{43/578…-51/201…}›] 80 match(30115) is out of range [lastIndex(30114)]. Was the raft log corrupted, truncated, or lost?
+asdasds
+`, "logs/foo.log", 0644, c.Node(2)))
+	if failed {
+		t.Error("boom")
+	}
+}