Merge pull request #159943 from rickystewart/backport24.3-159863

release-24.3: ci: add `github-actions-extended-ci` workflow and `maybe_stress` job

Author: rickystewart

.github/workflows/github-actions-extended-ci.yml

@@ -0,0 +1,56 @@
+name: GitHub Actions Extended CI
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    branches:
+      - "master"
+      - "release-*"
+      - "staging-*"
+      - "!release-1.0*"
+      - "!release-1.1*"
+      - "!release-2.0*"
+      - "!release-2.1*"
+      - "!release-19.1*"
+      - "!release-19.2*"
+      - "!release-20.1*"
+      - "!release-20.2*"
+      - "!release-21.1*"
+      - "!release-21.2*"
+      - "!release-22.1*"
+      - "!release-22.2*"
+      - "!release-23.1*"
+      - "!release-23.2*"
+      - "!staging-v22.2*"
+      - "!staging-v23.1*"
+      - "!staging-v23.2*"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+jobs:
+  maybe_stress:
+    if: github.event.pull_request.draft == false
+    runs-on: [self-hosted, ubuntu_2004]
+    timeout-minutes: 180
+    steps:
+      - name: calculate commit depth
+        run: echo COMMIT_DEPTH=$(echo '${{ github.event.pull_request.commits }} + 1' | bc) >> "$GITHUB_ENV"
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+          fetch-depth: ${{ env.COMMIT_DEPTH }}
+      - name: Fetch the base commit
+        run: git fetch origin ${{ github.event.pull_request.base.sha }} --depth 100
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
+      - run: ./build/github/get-engflow-keys.sh
+      - run: ./build/github/prepare-summarize-build.sh
+      - name: run tests
+        run: ./build/github/maybe-stress.sh ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }}
+      - name: upload build results
+        run: ./build/github/summarize-build.sh bes.bin
+        if: always()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: clean up
+        run: ./build/github/cleanup-engflow-keys.sh
+        if: always()

build/github/maybe-stress.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -euxo pipefail
+
+# Usage: must provide a base SHA and a PR head SHA as arguments.
+
+if [ -z "$1" ]
+then
+    echo 'Usage: build.sh BASESHA HEADSHA'
+    exit 1
+fi
+
+if [ -z "$2" ]
+then
+    echo 'Usage: build.sh BASESHA HEADSHA'
+    exit 1
+fi
+
+bazel build --config crosslinux //pkg/cmd/ci-stress \
+      --jobs 50 $(./build/github/engflow-args.sh)
+
+BASESHA="$1"
+HEADSHA="$2"
+
+set +e
+MERGEBASE=$(git merge-base $BASESHA $HEADSHA)
+set -e
+if [ -z "$MERGEBASE" ]
+then
+    echo 'Could not calculate merge base. You may have to rebase your in-progress PR.'
+    exit 1
+fi
+
+# NB: These jobs will run at a lower priority to ensure that the Essential CI
+# jobs (required to merge) will be minimally impacted.
+$(bazel info bazel-bin --config=crosslinux)/pkg/cmd/ci-stress/ci-stress_/ci-stress \
+    $MERGEBASE --config crosslinux --jobs 100 \
+    --remote_execution_priority=-1 \
+    --remote_download_minimal \
+    --bes_keywords ci-stress --config=use_ci_timeouts \
+    --build_event_binary_file=bes.bin \
+    $(./build/github/engflow-args.sh)

pkg/BUILD.bazel

@@ -136,6 +136,7 @@ ALL_TESTS = [
     "//pkg/cmd/bazci/githubpost:githubpost_test",
     "//pkg/cmd/bazci/testfilter:testfilter_test",
     "//pkg/cmd/bazci:bazci_lib_disallowed_imports_test",
+    "//pkg/cmd/ci-stress:ci-stress_test",
     "//pkg/cmd/cmpconn:cmpconn_test",
     "//pkg/cmd/cockroach:cockroach_lib_disallowed_imports_test",
     "//pkg/cmd/dev:dev_lib_disallowed_imports_test",
@@ -1087,6 +1088,9 @@ GO_TARGETS = [
     "//pkg/cmd/bazci/testfilter:testfilter_test",
     "//pkg/cmd/bazci:bazci",
     "//pkg/cmd/bazci:bazci_lib",
+    "//pkg/cmd/ci-stress:ci-stress",
+    "//pkg/cmd/ci-stress:ci-stress_lib",
+    "//pkg/cmd/ci-stress:ci-stress_test",
     "//pkg/cmd/cloudupload:cloudupload",
     "//pkg/cmd/cloudupload:cloudupload_lib",
     "//pkg/cmd/cmdutil:cmdutil",

pkg/cmd/ci-stress/BUILD.bazel

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+filegroup(
+    name = "testdata",
+    srcs = glob(["testdata/**"]),
+    visibility = ["//pkg/cmd/github-pull-request-make:__pkg__"],
+)
+
+go_library(
+    name = "ci-stress_lib",
+    srcs = ["main.go"],
+    importpath = "github.com/cockroachdb/cockroach/pkg/cmd/ci-stress",
+    visibility = ["//visibility:private"],
+    deps = ["//pkg/util/buildutil"],
+)
+
+go_binary(
+    name = "ci-stress",
+    embed = [":ci-stress_lib"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "ci-stress_test",
+    srcs = ["main_test.go"],
+    data = [":testdata"],  # keep
+    embed = [":ci-stress_lib"],
+    deps = ["//pkg/testutils/datapathutils"],
+)

pkg/cmd/ci-stress/main.go

@@ -0,0 +1,167 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+
+	"github.com/cockroachdb/cockroach/pkg/util/buildutil"
+)
+
+// It is a Test if there is a character after Test that is not a lower-case letter.
+const goTestStr = `func (Test[^a-z]\w*)\(.*\*testing\.TB?\) {$`
+
+var currentGoTestRE = regexp.MustCompile(`.*` + goTestStr)
+
+// getDiff returns the output of `git diff` from the given baseRef to the
+// current `HEAD`.
+func getDiff(ctx context.Context, baseRef string) (string, error) {
+	cmd := exec.CommandContext(ctx, "git", "diff", "--no-ext-diff", fmt.Sprintf("%s..HEAD", baseRef))
+	outputBytes, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("unable to get diff: git diff %s..HEAD [...]: %w", baseRef, err)
+	}
+	return strings.TrimSpace(string(outputBytes)), nil
+}
+
+// getPkgToTests parses a git-style diff and returns a mapping from directories
+// to affected tests in those directories in the given diff.
+func getPkgToTests(diff string) map[string][]string {
+	const newFilePrefix = "+++ b/"
+
+	ret := make(map[string][]string)
+
+	var curPkgName string
+
+	for _, line := range strings.Split(diff, "\n") {
+		if strings.HasPrefix(line, newFilePrefix) {
+			if strings.HasSuffix(line, ".go") {
+				curPkgName = filepath.Dir(strings.TrimPrefix(line, newFilePrefix))
+			} else {
+				curPkgName = ""
+			}
+		} else if currentGoTestRE.MatchString(line) && curPkgName != "" {
+			curTestName := ""
+			if !strings.HasPrefix(line, "-") {
+				curTestName = currentGoTestRE.ReplaceAllString(line, "$1")
+			}
+			if curPkgName != "" && curTestName != "" {
+				ret[curPkgName] = append(ret[curPkgName], curTestName)
+			}
+		}
+	}
+
+	// Sanity-check: Make sure there is a `BUILD.bazel` file in each pkg,
+	// or else it's not a real Go test. (Could be testdata for a different
+	// package, etc.) Don't do this in test builds however, as we would
+	// never find those files.
+	//
+	// We also take the opportunity to limit stressing to a constant
+	// number of packages in case this PR changed a ton of packages
+	// (find-and-replace, bulk changes, etc.)
+	if !buildutil.CrdbTestBuild {
+		taken := 0
+		for pkg := range ret {
+			_, err := os.Stat(filepath.Join(pkg, "BUILD.bazel"))
+			if taken >= 10 {
+				delete(ret, pkg)
+				continue
+			}
+			if err != nil && errors.Is(err, os.ErrNotExist) {
+				fmt.Printf("skipping testing package %s as we could not find a BUILD.bazel file in that directory\n", pkg)
+				delete(ret, pkg)
+				continue
+			} else if err != nil {
+				panic(err)
+			}
+			taken += 1
+		}
+	}
+
+	for _, tests := range ret {
+		slices.Sort(tests)
+	}
+	// De-duplicate.
+	for pkg := range ret {
+		ret[pkg] = slices.Compact(ret[pkg])
+	}
+
+	// We arbitrarily limit the number of tests per package. We randomize
+	// the tests selected.
+	for pkg := range ret {
+		const maxTests = 10
+		tests := ret[pkg]
+		if len(tests) > maxTests {
+			rand.Shuffle(len(tests), func(i, j int) {
+				tests[i], tests[j] = tests[j], tests[i]
+			})
+			tests = tests[:maxTests]
+			slices.Sort(tests)
+			ret[pkg] = tests
+		}
+	}
+
+	return ret
+}
+
+func runTests(ctx context.Context, pkgToTests map[string][]string, extraBazelArgs []string) error {
+	var testPackages []string
+	for pkg := range pkgToTests {
+		testPackages = append(testPackages, fmt.Sprintf("//%s:%s_test", pkg, filepath.Base(pkg)))
+	}
+	allTests := make(map[string]struct{})
+	for _, tests := range pkgToTests {
+		for _, test := range tests {
+			allTests[test] = struct{}{}
+		}
+	}
+	allTestsSlice := make([]string, 0, len(allTests))
+	for test := range allTests {
+		allTestsSlice = append(allTestsSlice, test)
+	}
+	slices.Sort(allTestsSlice)
+	testFilter := strings.Join(allTestsSlice, "|")
+	testFilter = "^(" + testFilter + ")$"
+	// Run each test multiple times.
+	bazelArgs := []string{"test", "--test_filter", testFilter, "--runs_per_test", "10"}
+	bazelArgs = append(bazelArgs, testPackages...)
+	bazelArgs = append(bazelArgs, extraBazelArgs...)
+	fmt.Printf("running `bazel` with args %+v\n", bazelArgs)
+	cmd := exec.CommandContext(ctx, "bazel", bazelArgs...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func mainImpl(baseRef string, bazelArgs []string) error {
+	ctx := context.Background()
+	diff, err := getDiff(ctx, baseRef)
+	if err != nil {
+		return err
+	}
+	pkgToTests := getPkgToTests(diff)
+	return runTests(ctx, pkgToTests, bazelArgs)
+}
+
+func main() {
+	if len(os.Args) == 1 {
+		panic("expected at least one argument (the ref of the base brach)")
+	}
+	baseRef := os.Args[1]
+	bazelArgs := os.Args[2:]
+	if err := mainImpl(baseRef, bazelArgs); err != nil {
+		panic(err)
+	}
+}

pkg/cmd/ci-stress/main_test.go

@@ -0,0 +1,62 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package main
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
+)
+
+func TestPkgToTests(t *testing.T) {
+	for filename, expPkgs := range map[string]map[string][]string{
+		datapathutils.TestDataPath(t, "skip.diff"): {
+			"pkg/ccl/storageccl": []string{"TestPutS3"},
+		},
+		datapathutils.TestDataPath(t, "modified.diff"): {
+			"pkg/ccl/crosscluster/physical": []string{"TestStreamingAutoReplan"},
+		},
+		datapathutils.TestDataPath(t, "removed.diff"): {},
+		datapathutils.TestDataPath(t, "not_go.diff"):  {},
+		datapathutils.TestDataPath(t, "new_test.diff"): {
+			"pkg/ccl/crosscluster/streamclient": []string{
+				"TestExternalConnectionClient",
+				"TestGetFirstActiveClientEmpty",
+			},
+		},
+		datapathutils.TestDataPath(t, "27595.diff"): {
+			"pkg/storage/closedts/container": []string{
+				"TestTwoNodes",
+			},
+			"pkg/storage/closedts/minprop": []string{
+				"TestTrackerConcurrentUse",
+			},
+			"pkg/storage/closedts/storage": []string{
+				"TestConcurrent",
+			},
+			"pkg/storage/closedts/transport": []string{
+				"TestTransportClientReceivesEntries",
+				"TestTransportConnectOnRequest",
+			},
+		},
+	} {
+		t.Run(filename, func(t *testing.T) {
+			content, err := os.ReadFile(filename)
+			if err != nil {
+				t.Fatal(err)
+			}
+			pkgToTests := getPkgToTests(string(content))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(pkgToTests, expPkgs) {
+				t.Errorf("expected %s, got %s", expPkgs, pkgToTests)
+			}
+		})
+	}
+}