Add testmask tool for conditional CI test execution (#4017)

## Changes

A new tool `testmask` under `tools/` that determines the set of Makefile
targets to run for testing.

The test jobs all depend on a job that runs this tool and use its output
to determine if they should run or skip. If a change only touches files
under `experimental/ssh`, for example, it won't trigger the main test
target. We can extend this to add more selective targets, only if we are
sure that there is no dependency between the files matched for the
target and other files in the repository. This, for now, relies on
convention rather than strict enforcement.

### How

Every file in the diff between the PR and the PR base is attributed to
one of the targets. If all files map to the same target, only that
target is run. If files map to multiple targets, multiple targets are
run. If there are files that cannot be mapped to a specific target, then
the main `test` target is run.

### Mapping to GitHub Actions

This approach uses a GitHub Actions test job for every target. This
results in a status check mark for each of them, making it easy to
eyeball which tests ran on a PR, which tests can on main, and when they
broke. Executing the same locally is straightforward because they are
all named after Makefile targets.

It applies to pull requests and merge queue checks. Scheduled runs and
main branch pushes always run the full test suite.

## Why

Not all tests need to run for every change. Being more selective reduces
build times and removes noise in PRs.

## Tests

The tests for this PR show it works.

A new test in `tools/testmask` confirms that the targets it maps to are
present in the Makefile.

PRs for each of the targets:
* #4023 
* #4024
* #4025
* #4026

Author: pietern

.github/actions/setup-build-environment/action.yml

@@ -0,0 +1,47 @@
+name: 'Setup Build Environment'
+description: 'Sets up the build environment with Go, Python, uv, and ruff'
+
+inputs:
+  cache-key:
+    description: 'Cache key identifier for Go cache'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Checkout repository and submodules
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+    - name: Create cache identifier
+      run: echo "${{ inputs.cache-key }}" > cache.txt
+      shell: bash
+
+    - name: Setup Go
+      uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+      with:
+        go-version-file: go.mod
+        cache-dependency-path: |
+          go.sum
+          cache.txt
+
+    - name: Setup Python
+      uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+      with:
+        python-version: '3.13'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
+      with:
+        version: "0.8.9"
+
+    - name: Install ruff (Python linter and formatter)
+      uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1
+      with:
+        version: "0.9.1"
+        args: "--version"
+
+    - name: Pull external libraries
+      run: |
+        go mod download
+        pip3 install wheel==0.45.1
+      shell: bash

.github/workflows/push.yml

@@ -31,8 +31,52 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: gh cache delete --all --repo databricks/cli || true
 
+  testmask:
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.mask1.outputs.targets || steps.mask2.outputs.targets || steps.mask3.outputs.targets }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
+        with:
+          go-version-file: tools/go.mod
+
+      - name: Run testmask (pull requests)
+        if: ${{ github.event_name == 'pull_request' }}
+        id: mask1
+        working-directory: tools/testmask
+        run: |
+          go run . ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.base.sha }} | tee output.json
+          echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT
+
+      - name: Run testmask (merge group)
+        if: ${{ github.event_name == 'merge_group' }}
+        id: mask2
+        working-directory: tools/testmask
+        run: |
+          go run . ${{ github.event.merge_group.head_sha }} ${{ github.event.merge_group.base_sha }} | tee output.json
+          echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT
+
+      - name: Run testmask (other events)
+        if: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }}
+        id: mask3
+        working-directory: tools/testmask
+        run: |
+          # Always run all tests
+          echo "targets=[\"test\"]" >> $GITHUB_OUTPUT
+
   tests:
-    needs: cleanups
+    needs:
+      - cleanups
+      - testmask
+
+    # Only run if the target is in the list of targets from testmask
+    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }}
     runs-on: ${{ matrix.os }}
 
     strategy:
@@ -56,37 +100,10 @@ jobs:
       - name: Checkout repository and submodules
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - name: Create deployment-specific cache identifier
-        run: echo "${{ matrix.deployment }}" > deployment-type.txt
-
-      - name: Setup Go
-        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
-        with:
-          go-version-file: go.mod
-          cache-dependency-path: |
-            go.sum
-            deployment-type.txt
-
-      - name: Setup Python
-        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
         with:
-          python-version: '3.13'
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
-        with:
-          version: "0.8.9"
-
-      - name: Install ruff (Python linter and formatter)
-        uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1
-        with:
-          version: "0.9.1"
-          args: "--version"
-
-      - name: Pull external libraries
-        run: |
-          go mod download
-          pip3 install wheel==0.45.1
+          cache-key: test-${{ matrix.deployment }}
 
       - name: Run tests without coverage
         # We run tests without coverage on PR, merge_group, and schedule because we don't make use of coverage information
@@ -107,6 +124,131 @@ jobs:
       - name: Analyze slow tests
         run: make slowest
 
+  test-exp-aitools:
+    needs:
+      - cleanups
+      - testmask
+
+    # Only run if the target is in the list of targets from testmask
+    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-aitools') }}
+    name: "make test-exp-aitools"
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+          - windows-latest
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-exp-aitools
+
+      - name: Run tests
+        run: |
+          make test-exp-aitools
+
+  test-exp-apps-mcp:
+    needs:
+      - cleanups
+      - testmask
+
+    # Only run if the target is in the list of targets from testmask
+    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-apps-mcp') }}
+    name: "make test-exp-apps-mcp"
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+          # The Windows tests are broken; see https://github.com/databricks/cli/pull/4024.
+          # - windows-latest
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-exp-apps-mcp
+
+      - name: Run tests
+        run: |
+          make test-exp-apps-mcp
+
+  test-exp-ssh:
+    needs:
+      - cleanups
+      - testmask
+
+    # Only run if the target is in the list of targets from testmask
+    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh') }}
+    name: "make test-exp-ssh"
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+          - windows-latest
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-exp-ssh
+
+      - name: Run tests
+        run: |
+          make test-exp-ssh
+
+  test-pipelines:
+    needs:
+      - cleanups
+      - testmask
+
+    # Only run if the target is in the list of targets from testmask
+    if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-pipelines') }}
+    name: "make test-pipelines"
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macos-latest
+          - ubuntu-latest
+          - windows-latest
+
+    steps:
+      - name: Checkout repository and submodules
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup build environment
+        uses: ./.github/actions/setup-build-environment
+        with:
+          cache-key: test-pipelines
+
+      - name: Run tests
+        run: |
+          make test-pipelines
+
   validate-generated-is-up-to-date:
     needs: cleanups
     runs-on: ubuntu-latest

Makefile

@@ -1,7 +1,10 @@
 default: checks fmt lint
 
-# gotestsum: when go test args are used with --rerun-fails the list of packages to test must be specified by the --packages flag
-PACKAGES=--packages "./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/aitools/... ./experimental/ssh/... ."
+# Default packages to test (all)
+TEST_PACKAGES = ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/aitools/... ./experimental/ssh/... .
+
+# Default acceptance test filter (all)
+ACCEPTANCE_TEST_FILTER = ""
 
 GO_TOOL ?= go tool -modfile=tools/go.mod
 GOTESTSUM_FORMAT ?= pkgname-and-test-fails
@@ -55,11 +58,24 @@ links:
 # Checks other than 'fmt' and 'lint'; these are fast, so can be run first
 checks: tidy ws links
 
-test:
-	${GOTESTSUM_CMD} ${PACKAGES} -- -timeout=${LOCAL_TIMEOUT} -short
 
-test-slow:
-	${GOTESTSUM_CMD} ${PACKAGES} -- -timeout=${LOCAL_TIMEOUT}
+# Run short unit and acceptance tests (testing.Short() is true).
+test: test-unit test-acc
+
+# Run all unit and acceptance tests.
+test-slow: test-slow-unit test-slow-acc
+
+test-unit:
+	${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} -short
+
+test-slow-unit:
+	${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT}
+
+test-acc:
+	${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -short -run ${ACCEPTANCE_TEST_FILTER}
+
+test-slow-acc:
+	${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -run ${ACCEPTANCE_TEST_FILTER}
 
 # Updates acceptance test output (local tests)
 test-update:
@@ -82,7 +98,7 @@ slowest:
 
 cover:
 	rm -fr ./acceptance/build/cover/
-	VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} ${PACKAGES} -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT}
+	VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} --packages ${TEST_PACKAGES} -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT}
 	rm -fr ./acceptance/build/cover-merged/
 	mkdir -p acceptance/build/cover-merged/
 	go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
@@ -150,4 +166,16 @@ generate:
 	$(GENKIT_BINARY) update-sdk
 
 
-.PHONY: lint lintfull tidy lintcheck fmt fmtfull test cover showcover build snapshot snapshot-release schema integration integration-short acc-cover acc-showcover docs ws wsfix links checks test-update test-update-templates test-update-aws test-update-all generate-validation
+.PHONY: lint lintfull tidy lintcheck fmt fmtfull test test-unit test-acc test-slow test-slow-unit test-slow-acc cover showcover build snapshot snapshot-release schema integration integration-short acc-cover acc-showcover docs ws wsfix links checks test-update test-update-templates test-update-aws test-update-all generate-validation
+
+test-exp-aitools:
+	make test TEST_PACKAGES="./experimental/aitools/..." ACCEPTANCE_TEST_FILTER="TestAccept/idontexistyet/aitools"
+
+test-exp-apps-mcp:
+	make test TEST_PACKAGES="./experimental/apps-mcp/..." ACCEPTANCE_TEST_FILTER="TestAccept/idontexistyet/apps-mcp"
+
+test-exp-ssh:
+	make test TEST_PACKAGES="./experimental/ssh/..." ACCEPTANCE_TEST_FILTER="TestAccept/ssh"
+
+test-pipelines:
+	make test TEST_PACKAGES="./cmd/pipelines/..." ACCEPTANCE_TEST_FILTER="TestAccept/pipelines"

tools/testmask/git.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+// GetChangedFiles returns the list of files changed between two git refs.
+func GetChangedFiles(headRef, baseRef string) ([]string, error) {
+	cmd := exec.Command("git", "diff", "--name-only", baseRef, headRef)
+	output, err := cmd.Output()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get diff between %s and %s: %w", baseRef, headRef, err)
+	}
+
+	lines := strings.Split(string(output), "\n")
+
+	// Drop the last line (always empty)
+	if len(lines) > 0 {
+		lines = lines[:len(lines)-1]
+	}
+
+	return lines, nil
+}