Skip to content

Commit 6038332

Browse files
shreyas-goenkashreyas-goenka
committed
update
2 parents 50fc3ac + 7a84cf1 commit 6038332

File tree

291 files changed

+10842
-1876
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

291 files changed

+10842
-1876
lines changed

‎.codegen/_openapi_sha‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
59c4c0f3d5f0ef00cd5350b5674e941a7606d91a
1+
8f5eedbc991c4f04ce1284406577b0c92d59a224

‎.github/ISSUE_TEMPLATE/direct-issue.md‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
name: Bug report for direct deployment engine for DABs
3+
about: Use this to report an issue with direct deployment engine in Databricks Asset Bundles.
4+
labels: DABs engine/direct Bug
5+
title: ''
6+
---
7+
8+
### Describe the issue
9+
A clear and concise description of what the issue is
10+
11+
### Configuration
12+
Please provide a minimal reproducible configuration for the issue
13+
14+
### Steps to reproduce the behavior
15+
Please list the steps required to reproduce the issue, for example:
16+
1. Run `databricks bundle deploy ...`
17+
2. Run `databricks bundle run ...`
18+
3. See error
19+
20+
### Expected Behavior
21+
Clear and concise description of what should have happened
22+
23+
### Actual Behavior
24+
Clear and concise description of what actually happened
25+
26+
### OS and CLI version
27+
Please provide the version of the CLI (eg: v0.1.2) and the operating system (eg: windows). You can run databricks --version to get the version of your Databricks CLI
28+
29+
### Is this a regression?
30+
Did this work in a previous version of the CLI? If so, which versions did you try?
31+
32+
### Detailed plan
33+
34+
If relevant, please provide redacted output of `bundle plan -o json -t <your target>`. The plan includes reasons for a given action per resource.
35+
36+
### Debug Logs
37+
Output logs if you run the command with debug logs enabled. Example: databricks bundle deploy --log-level=debug. Redact if needed

‎.github/actions/setup-build-environment/action.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: 'Setup Build Environment'
2+
description: 'Sets up the build environment with Go, Python, uv, and ruff'
3+
4+
inputs:
5+
cache-key:
6+
description: 'Cache key identifier for Go cache'
7+
required: true
8+
9+
runs:
10+
using: 'composite'
11+
steps:
12+
- name: Checkout repository and submodules
13+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
14+
15+
- name: Create cache identifier
16+
run: echo "${{ inputs.cache-key }}" > cache.txt
17+
shell: bash
18+
19+
- name: Setup Go
20+
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
21+
with:
22+
go-version-file: go.mod
23+
cache-dependency-path: |
24+
go.sum
25+
cache.txt
26+
27+
- name: Setup Python
28+
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
29+
with:
30+
python-version: '3.13'
31+
32+
- name: Install uv
33+
uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
34+
with:
35+
version: "0.8.9"
36+
37+
- name: Install ruff (Python linter and formatter)
38+
uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1
39+
with:
40+
version: "0.9.1"
41+
args: "--version"
42+
43+
- name: Pull external libraries
44+
run: |
45+
go mod download
46+
pip3 install wheel==0.45.1
47+
shell: bash

‎.github/workflows/push.yml‎

Lines changed: 173 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,52 @@ jobs:
3131
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3232
run: gh cache delete --all --repo databricks/cli || true
3333

34+
testmask:
35+
runs-on: ubuntu-latest
36+
outputs:
37+
targets: ${{ steps.mask1.outputs.targets || steps.mask2.outputs.targets || steps.mask3.outputs.targets }}
38+
steps:
39+
- name: Checkout repository
40+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
41+
with:
42+
fetch-depth: 0
43+
44+
- name: Setup Go
45+
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
46+
with:
47+
go-version-file: tools/go.mod
48+
49+
- name: Run testmask (pull requests)
50+
if: ${{ github.event_name == 'pull_request' }}
51+
id: mask1
52+
working-directory: tools/testmask
53+
run: |
54+
go run . ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.base.sha }} | tee output.json
55+
echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT
56+
57+
- name: Run testmask (merge group)
58+
if: ${{ github.event_name == 'merge_group' }}
59+
id: mask2
60+
working-directory: tools/testmask
61+
run: |
62+
go run . ${{ github.event.merge_group.head_sha }} ${{ github.event.merge_group.base_sha }} | tee output.json
63+
echo "targets=$(jq -c '.' output.json)" >> $GITHUB_OUTPUT
64+
65+
- name: Run testmask (other events)
66+
if: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }}
67+
id: mask3
68+
working-directory: tools/testmask
69+
run: |
70+
# Always run all tests
71+
echo "targets=[\"test\"]" >> $GITHUB_OUTPUT
72+
3473
tests:
35-
needs: cleanups
74+
needs:
75+
- cleanups
76+
- testmask
77+
78+
# Only run if the target is in the list of targets from testmask
79+
if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test') }}
3680
runs-on: ${{ matrix.os }}
3781

3882
strategy:
@@ -56,37 +100,10 @@ jobs:
56100
- name: Checkout repository and submodules
57101
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
58102

59-
- name: Create deployment-specific cache identifier
60-
run: echo "${{ matrix.deployment }}" > deployment-type.txt
61-
62-
- name: Setup Go
63-
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
64-
with:
65-
go-version-file: go.mod
66-
cache-dependency-path: |
67-
go.sum
68-
deployment-type.txt
69-
70-
- name: Setup Python
71-
uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
103+
- name: Setup build environment
104+
uses: ./.github/actions/setup-build-environment
72105
with:
73-
python-version: '3.13'
74-
75-
- name: Install uv
76-
uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
77-
with:
78-
version: "0.8.9"
79-
80-
- name: Install ruff (Python linter and formatter)
81-
uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1
82-
with:
83-
version: "0.9.1"
84-
args: "--version"
85-
86-
- name: Pull external libraries
87-
run: |
88-
go mod download
89-
pip3 install wheel==0.45.1
106+
cache-key: test-${{ matrix.deployment }}
90107

91108
- name: Run tests without coverage
92109
# We run tests without coverage on PR, merge_group, and schedule because we don't make use of coverage information
@@ -107,6 +124,131 @@ jobs:
107124
- name: Analyze slow tests
108125
run: make slowest
109126

127+
test-exp-aitools:
128+
needs:
129+
- cleanups
130+
- testmask
131+
132+
# Only run if the target is in the list of targets from testmask
133+
if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-aitools') }}
134+
name: "make test-exp-aitools"
135+
runs-on: ${{ matrix.os }}
136+
137+
strategy:
138+
fail-fast: false
139+
matrix:
140+
os:
141+
- macos-latest
142+
- ubuntu-latest
143+
- windows-latest
144+
145+
steps:
146+
- name: Checkout repository and submodules
147+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
148+
149+
- name: Setup build environment
150+
uses: ./.github/actions/setup-build-environment
151+
with:
152+
cache-key: test-exp-aitools
153+
154+
- name: Run tests
155+
run: |
156+
make test-exp-aitools
157+
158+
test-exp-apps-mcp:
159+
needs:
160+
- cleanups
161+
- testmask
162+
163+
# Only run if the target is in the list of targets from testmask
164+
if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-apps-mcp') }}
165+
name: "make test-exp-apps-mcp"
166+
runs-on: ${{ matrix.os }}
167+
168+
strategy:
169+
fail-fast: false
170+
matrix:
171+
os:
172+
- macos-latest
173+
- ubuntu-latest
174+
# The Windows tests are broken; see https://github.com/databricks/cli/pull/4024.
175+
# - windows-latest
176+
177+
steps:
178+
- name: Checkout repository and submodules
179+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
180+
181+
- name: Setup build environment
182+
uses: ./.github/actions/setup-build-environment
183+
with:
184+
cache-key: test-exp-apps-mcp
185+
186+
- name: Run tests
187+
run: |
188+
make test-exp-apps-mcp
189+
190+
test-exp-ssh:
191+
needs:
192+
- cleanups
193+
- testmask
194+
195+
# Only run if the target is in the list of targets from testmask
196+
if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-exp-ssh') }}
197+
name: "make test-exp-ssh"
198+
runs-on: ${{ matrix.os }}
199+
200+
strategy:
201+
fail-fast: false
202+
matrix:
203+
os:
204+
- macos-latest
205+
- ubuntu-latest
206+
- windows-latest
207+
208+
steps:
209+
- name: Checkout repository and submodules
210+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
211+
212+
- name: Setup build environment
213+
uses: ./.github/actions/setup-build-environment
214+
with:
215+
cache-key: test-exp-ssh
216+
217+
- name: Run tests
218+
run: |
219+
make test-exp-ssh
220+
221+
test-pipelines:
222+
needs:
223+
- cleanups
224+
- testmask
225+
226+
# Only run if the target is in the list of targets from testmask
227+
if: ${{ contains(fromJSON(needs.testmask.outputs.targets), 'test-pipelines') }}
228+
name: "make test-pipelines"
229+
runs-on: ${{ matrix.os }}
230+
231+
strategy:
232+
fail-fast: false
233+
matrix:
234+
os:
235+
- macos-latest
236+
- ubuntu-latest
237+
- windows-latest
238+
239+
steps:
240+
- name: Checkout repository and submodules
241+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
242+
243+
- name: Setup build environment
244+
uses: ./.github/actions/setup-build-environment
245+
with:
246+
cache-key: test-pipelines
247+
248+
- name: Run tests
249+
run: |
250+
make test-pipelines
251+
110252
validate-generated-is-up-to-date:
111253
needs: cleanups
112254
runs-on: ubuntu-latest

‎Makefile‎

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
default: checks fmt lint
22

3-
# gotestsum: when go test args are used with --rerun-fails the list of packages to test must be specified by the --packages flag
4-
PACKAGES=--packages "./acceptance/... ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/aitools/... ./experimental/ssh/... ."
3+
# Default packages to test (all)
4+
TEST_PACKAGES = ./acceptance/internal ./libs/... ./internal/... ./cmd/... ./bundle/... ./experimental/aitools/... ./experimental/ssh/... .
5+
6+
# Default acceptance test filter (all)
7+
ACCEPTANCE_TEST_FILTER = ""
58

69
GO_TOOL ?= go tool -modfile=tools/go.mod
710
GOTESTSUM_FORMAT ?= pkgname-and-test-fails
@@ -55,11 +58,24 @@ links:
5558
# Checks other than 'fmt' and 'lint'; these are fast, so can be run first
5659
checks: tidy ws links
5760

58-
test:
59-
${GOTESTSUM_CMD} ${PACKAGES} -- -timeout=${LOCAL_TIMEOUT} -short
6061

61-
test-slow:
62-
${GOTESTSUM_CMD} ${PACKAGES} -- -timeout=${LOCAL_TIMEOUT}
62+
# Run short unit and acceptance tests (testing.Short() is true).
63+
test: test-unit test-acc
64+
65+
# Run all unit and acceptance tests.
66+
test-slow: test-slow-unit test-slow-acc
67+
68+
test-unit:
69+
${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT} -short
70+
71+
test-slow-unit:
72+
${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -timeout=${LOCAL_TIMEOUT}
73+
74+
test-acc:
75+
${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -short -run ${ACCEPTANCE_TEST_FILTER}
76+
77+
test-slow-acc:
78+
${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -run ${ACCEPTANCE_TEST_FILTER}
6379

6480
# Updates acceptance test output (local tests)
6581
test-update:
@@ -82,7 +98,8 @@ slowest:
8298

8399
cover:
84100
rm -fr ./acceptance/build/cover/
85-
VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} ${PACKAGES} -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT}
101+
VERBOSE_TEST=1 ${GOTESTSUM_CMD} --packages "${TEST_PACKAGES}" -- -coverprofile=coverage.txt -timeout=${LOCAL_TIMEOUT}
102+
VERBOSE_TEST=1 CLI_GOCOVERDIR=build/cover ${GOTESTSUM_CMD} --packages ./acceptance/... -- -timeout=${LOCAL_TIMEOUT} -run ${ACCEPTANCE_TEST_FILTER}
86103
rm -fr ./acceptance/build/cover-merged/
87104
mkdir -p acceptance/build/cover-merged/
88105
go tool covdata merge -i $$(printf '%s,' acceptance/build/cover/* | sed 's/,$$//') -o acceptance/build/cover-merged/
@@ -150,4 +167,16 @@ generate:
150167
$(GENKIT_BINARY) update-sdk
151168

152169

153-
.PHONY: lint lintfull tidy lintcheck fmt fmtfull test cover showcover build snapshot snapshot-release schema integration integration-short acc-cover acc-showcover docs ws wsfix links checks test-update test-update-templates test-update-aws test-update-all generate-validation
170+
.PHONY: lint lintfull tidy lintcheck fmt fmtfull test test-unit test-acc test-slow test-slow-unit test-slow-acc cover showcover build snapshot snapshot-release schema integration integration-short acc-cover acc-showcover docs ws wsfix links checks test-update test-update-templates test-update-aws test-update-all generate-validation
171+
172+
test-exp-aitools:
173+
make test TEST_PACKAGES="./experimental/aitools/..." ACCEPTANCE_TEST_FILTER="TestAccept/idontexistyet/aitools"
174+
175+
test-exp-apps-mcp:
176+
make test TEST_PACKAGES="./experimental/apps-mcp/..." ACCEPTANCE_TEST_FILTER="TestAccept/idontexistyet/apps-mcp"
177+
178+
test-exp-ssh:
179+
make test TEST_PACKAGES="./experimental/ssh/..." ACCEPTANCE_TEST_FILTER="TestAccept/ssh"
180+
181+
test-pipelines:
182+
make test TEST_PACKAGES="./cmd/pipelines/..." ACCEPTANCE_TEST_FILTER="TestAccept/pipelines"

‎NEXT_CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,6 @@
1010

1111
### Bundles
1212
* Add support for alerts to DABs ([#4004](https://github.com/databricks/cli/pull/4004))
13+
* Allow `file://` URIs in job libraries to reference runtime filesystem paths (e.g., JARs pre-installed on clusters via init scripts). These paths are no longer treated as local files to upload. ([#3884](https://github.com/databricks/cli/pull/3884))
1314

1415
### API Changes

0 commit comments

Comments
 (0)