Merge #150774

150774: asim: run no_rand tests in parallel r=tbg a=tbg

Each simulation is single-threaded, so it makes sense to evaluate different
testfiles in parallel.

A complication is that each simulation logs using our default logging library,
and since that is a singleton, we can't provide each simulation with its own
logger. When a subtest is a parallel test (as is the case here), it waits for
the parent test to return before running. This means that the `log.Scope` in
`TestDataDriven` would be cleaned up by the time the parallel subtests would
actually start simulating.

I introduced a refcounting version of `log.Scope` instead, so the scope will
be closed only when the refcount drops to zero. Each subtest will log the
scope it's using, so the logs can be found, and should then be filtered by
the log tag which is specific to the simulation of interest.

```
./dev test pkg/kv/kvserver/asim/tests  -f 'TestDataDriven'  --ignore-cache --rewrite  -v -- --test_env COCKROACH_RUN_ASIM_TESTS=true
//pkg/kv/kvserver/asim/tests:tests_test                                  PASSED in 202.1s # before
//pkg/kv/kvserver/asim/tests:tests_test                                  PASSED in 60.6s  # after
```

Epic: CRDB-25222


Co-authored-by: Tobias Grieger <[email protected]>

Author: craig[bot]

pkg/kv/kvserver/asim/tests/BUILD.bazel

@@ -56,6 +56,7 @@ go_test(
         "//pkg/util/leaktest",
         "//pkg/util/log",
         "@com_github_cockroachdb_datadriven//:datadriven",
+        "@com_github_cockroachdb_logtags//:logtags",
         "@com_github_stretchr_testify//require",
         "@org_gonum_v1_plot//:plot",
         "@org_gonum_v1_plot//plotter",

pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/datadriven"
+	"github.com/cockroachdb/logtags"
 	"github.com/stretchr/testify/require"
 	"gonum.org/v1/plot"
 	"gonum.org/v1/plot/plotter"
@@ -182,17 +183,27 @@ var runAsimTests = envutil.EnvOrDefaultBool("COCKROACH_RUN_ASIM_TESTS", false)
 //     ..US_3
 //     ....└── [11 12 13 14 15]
 func TestDataDriven(t *testing.T) {
-	defer leaktest.AfterTest(t)()
-	defer log.Scope(t).Close(t)
 	skip.UnderDuressWithIssue(t, 149875)
-	ctx := context.Background()
+	leakTestAfter := leaktest.AfterTest(t)
 	dir := datapathutils.TestDataPath(t, "non_rand")
+
+	// NB: we must use t.Cleanup instead of deferring this
+	// cleanup in the main test due to the use of t.Parallel
+	// below.
+	// See https://github.com/golang/go/issues/31651.
+	scope := log.Scope(t)
+	t.Cleanup(func() {
+		scope.Close(t)
+		leakTestAfter()
+	})
 	datadriven.Walk(t, dir, func(t *testing.T, path string) {
-		if filepath.Ext(path) == ".png" {
-			// TODO(tbg): make this != "txt" once all test files have that suffix.
+		if filepath.Ext(path) != ".txt" {
 			return
 		}
-		plotNum := 0
+		ctx := logtags.AddTag(context.Background(), "name", filepath.Base(path))
+		// The inline comment below is required for TestLint/TestTParallel.
+		// We use t.Cleanup to work around the issue this lint is trying to prevent.
+		t.Parallel() // SAFE FOR TESTING
 		const defaultKeyspace = 10000
 		loadGen := gen.MultiLoad{}
 		var clusterGen gen.ClusterGen
@@ -201,7 +212,8 @@ func TestDataDriven(t *testing.T) {
 		eventGen := gen.NewStaticEventsWithNoEvents()
 		assertions := []assertion.SimulationAssertion{}
 		var stateStrAcrossSamples []string
-		runs := []history.History{}
+		var runs []history.History
+		plotNum := 0
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
 			defer func() {
 				require.Empty(t, d.CmdArgs, "leftover arguments for %s", d.Cmd)