Merge #150167

150167: nightlies: run `TestDataDriven` once under `cockroach_nightly_extra_stress_impl.sh` r=tbg,stevendanna a=wenyihu6

**nightlies: run `TestDataDriven` once under `cockroach_nightly_extra_stress_impl.sh`**

`TestDataDriven` in `pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go` is
slow and doesn't add much value on CI, as it's datadriven and mainly checks for
determinism. To reduce CI load, this commit adds a `skip_under_ci` flag to skip
specific test files that take a long time to run during standard CI. To retain
test coverage, we’ll continue running them once with a longer timeout in the
Nightly Cockroach Extra Stress pipeline we recently introduced.

Epic: none
Release note: none

---

**asim: add `skip_under_ci` for some tests in `pkg/kv/kvserver/asim/tests`**

This commit adds the skip_under_ci flag to the `example_rebalancing`,
`example_fulldisk`, `example_liveness`, `example_splitting`, and `example_zone_config`
test files, as they tend to run for a longer duration. We plan to extend this
flag to additional long-running tests in the future.

Epic: none
Release note: none


Co-authored-by: wenyihu6 <[email protected]>

Author: craig[bot]

build/teamcity/cockroach/nightlies/cockroach_nightly_extra_stress_impl.sh

@@ -55,6 +55,17 @@ bazel build //pkg/cmd/bazci
 BAZEL_BIN=$(bazel info bazel-bin)
 
 exit_status=0
+
+# Run TestDataDriven from the asim package once with COCKROACH_RUN_ASIM_TESTS enabled.
+$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci -- test //pkg/kv/kvserver/asim/tests:tests_test \
+                                      --config=$BAZEL_CONFIG \
+                                      --test_env TC_SERVER_URL \
+                                      --test_env COCKROACH_RUN_ASIM_TESTS=true \
+                                      --test_filter="^TestDataDriven$" \
+                                      --test_timeout=3600 \
+    || exit_status=$?
+
+# Run the rest of the tests under stress.
 $BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci -- test $TEST_PACKAGE_TARGETS \
                                       --config=$BAZEL_CONFIG \
                                       --test_env TC_SERVER_URL \

pkg/kv/kvserver/asim/tests/BUILD.bazel

@@ -50,6 +50,9 @@ go_test(
         "//pkg/spanconfig/spanconfigtestutils",
         "//pkg/testutils/datapathutils",
         "//pkg/testutils/skip",
+        "//pkg/util/envutil",
+        "//pkg/util/leaktest",
+        "//pkg/util/log",
         "@com_github_cockroachdb_datadriven//:datadriven",
         "@com_github_guptarohit_asciigraph//:asciigraph",
         "@com_github_stretchr_testify//require",

pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go

@@ -25,11 +25,16 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
+	"github.com/cockroachdb/cockroach/pkg/util/envutil"
+	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
+	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/datadriven"
 	"github.com/guptarohit/asciigraph"
 	"github.com/stretchr/testify/require"
 )
 
+var runAsimTests = envutil.EnvOrDefaultBool("COCKROACH_RUN_ASIM_TESTS", false)
+
 // TestDataDriven is a data-driven test for the allocation system using the
 // simulator. It gives contributors a way to understand how a cluster reacts to
 // different settings, load and configuration. In addition, it may be used for
@@ -168,6 +173,8 @@ import (
 //     ..US_3
 //     ....└── [11 12 13 14 15]
 func TestDataDriven(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
 	skip.UnderDuressWithIssue(t, 149875)
 	ctx := context.Background()
 	dir := datapathutils.TestDataPath(t, "non_rand")
@@ -186,6 +193,12 @@ func TestDataDriven(t *testing.T) {
 				require.Empty(t, d.CmdArgs, "leftover arguments for %s", d.Cmd)
 			}()
 			switch d.Cmd {
+			case "skip_under_ci":
+				if !runAsimTests {
+					t.Logf("skipping %s: skip_under_ci and COCKROACH_RUN_ASIM_TESTS is not set", path)
+					t.SkipNow()
+				}
+				return ""
 			case "gen_load":
 				var rwRatio, rate = 0.0, 0.0
 				var minBlock, maxBlock = 1, 1
@@ -372,6 +385,7 @@ func TestDataDriven(t *testing.T) {
 
 				return ""
 			case "eval":
+				t.Logf("running eval for %s", path)
 				samples := 1
 				seed := rand.Int63()
 				duration := 30 * time.Minute

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_fulldisk.txt

@@ -1,3 +1,6 @@
+skip_under_ci
+----
+
 # Set every store's capacity to 512 GiB, we will later adjust just one store to
 # have less free capacity.
 gen_cluster nodes=5 store_byte_capacity=549755813888

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_liveness.txt

@@ -1,3 +1,6 @@
+skip_under_ci
+----
+
 # This example sets n7 to dead initially and n5 to decommissioning after 2
 # minutes. The output of replicas per store is then plotted.
 #

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing.txt

@@ -1,3 +1,6 @@
+skip_under_ci
+----
+
 # Walk through the basics of the datadriven syntax. Create a state generator
 # where there are 7 stores, 7 ranges and initially the replicas are placed
 # following a skewed distribution (where s1 has the most replicas, s2 has half

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_splitting.txt

@@ -1,3 +1,6 @@
+skip_under_ci
+----
+
 # Explore how load based and sized based splitting occur in isolation. In this
 # example, there is only one store so no rebalancing activity should occur. 
 gen_cluster nodes=1

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_zone_config.txt

@@ -1,3 +1,6 @@
+skip_under_ci
+----
+
 # This test applies a configuration that prioritizes zone constraints, favoring
 # the US_East region. As a result, we expect a majority of replicas to be
 # distributed across stores numbered 1-12, all within the US_East region. The