Skip to content

Commit ef5f905

Browse files
denikdenik
committed
Add integration test for default Python bundle initialization
1 parent 3c3c172 commit ef5f905

File tree

10 files changed

+638
-2
lines changed

10 files changed

+638
-2
lines changed

‎go.mod‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
module github.com/databricks/cli
22

3-
go 1.23
3+
go 1.23.0
44

5-
toolchain go1.23.2
5+
toolchain go1.23.3
66

77
require (
88
github.com/Masterminds/semver/v3 v3.3.1 // MIT
99
github.com/briandowns/spinner v1.23.1 // Apache 2.0
1010
github.com/databricks/databricks-sdk-go v0.54.0 // Apache 2.0
11+
github.com/elliotchance/orderedmap/v3 v3.0.0 // MIT
1112
github.com/fatih/color v1.18.0 // MIT
1213
github.com/ghodss/yaml v1.0.0 // MIT + NOTICE
1314
github.com/google/uuid v1.6.0 // BSD-3-Clause
1415
github.com/hashicorp/go-version v1.7.0 // MPL 2.0
1516
github.com/hashicorp/hc-install v0.9.0 // MPL 2.0
1617
github.com/hashicorp/terraform-exec v0.21.0 // MPL 2.0
1718
github.com/hashicorp/terraform-json v0.23.0 // MPL 2.0
19+
github.com/hexops/gotextdiff v1.0.3 // BSD 3-Clause "New" or "Revised" License
1820
github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause
1921
github.com/mattn/go-isatty v0.0.20 // MIT
2022
github.com/nwidger/jsoncolor v0.3.2 // MIT
@@ -23,6 +25,7 @@ require (
2325
github.com/spf13/cobra v1.8.1 // Apache 2.0
2426
github.com/spf13/pflag v1.0.5 // BSD-3-Clause
2527
github.com/stretchr/testify v1.10.0 // MIT
28+
github.com/wI2L/jsondiff v0.6.1 // MIT
2629
golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
2730
golang.org/x/mod v0.22.0
2831
golang.org/x/oauth2 v0.24.0
@@ -56,6 +59,10 @@ require (
5659
github.com/mattn/go-colorable v0.1.13 // indirect
5760
github.com/pmezard/go-difflib v1.0.0 // indirect
5861
github.com/stretchr/objx v0.5.2 // indirect
62+
github.com/tidwall/gjson v1.18.0 // indirect
63+
github.com/tidwall/match v1.1.1 // indirect
64+
github.com/tidwall/pretty v1.2.1 // indirect
65+
github.com/tidwall/sjson v1.2.5 // indirect
5966
github.com/zclconf/go-cty v1.15.0 // indirect
6067
go.opencensus.io v0.24.0 // indirect
6168
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect

‎go.sum‎

Lines changed: 16 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎integration/bundle/init_default_python_test.go‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package bundle_test
2+
3+
import (
4+
"encoding/json"
5+
"os"
6+
"path/filepath"
7+
"testing"
8+
9+
"github.com/databricks/cli/integration/internal/acc"
10+
"github.com/databricks/cli/internal/testcli"
11+
"github.com/databricks/cli/internal/testutil"
12+
"github.com/databricks/cli/libs/python/pythontest"
13+
"github.com/stretchr/testify/require"
14+
)
15+
16+
var pythonVersions = []string{
17+
"3.8",
18+
"3.9",
19+
"3.10",
20+
"3.11",
21+
"3.12",
22+
"3.13",
23+
}
24+
25+
var pythonVersionsShort = []string{
26+
"3.9",
27+
"3.12",
28+
}
29+
30+
var extraInstalls = map[string][]string{
31+
"3.12": {"setuptools"},
32+
"3.13": {"setuptools"},
33+
}
34+
35+
func TestDefaultPython(t *testing.T) {
36+
versions := pythonVersions
37+
if testing.Short() {
38+
versions = pythonVersionsShort
39+
}
40+
41+
for _, pythonVersion := range versions {
42+
t.Run(pythonVersion, func(t *testing.T) {
43+
testDefaultPython(t, pythonVersion)
44+
})
45+
}
46+
}
47+
48+
func testDefaultPython(t *testing.T, pythonVersion string) {
49+
ctx, wt := acc.WorkspaceTest(t)
50+
51+
uniqueProjectId := testutil.RandomName("")
52+
ctx, replacements := testcli.WithReplacementsMap(ctx)
53+
replacements.Set(uniqueProjectId, "$UNIQUE_PRJ")
54+
55+
testcli.PrepareReplacements(t, replacements, wt.W)
56+
57+
user, err := wt.W.CurrentUser.Me(ctx)
58+
require.NoError(t, err)
59+
if user != nil {
60+
testcli.PrepareReplacementsUser(t, replacements, *user)
61+
}
62+
63+
tmpDir1 := pythontest.RequirePythonVENV(t, ctx, pythonVersion, true)
64+
extras, ok := extraInstalls[pythonVersion]
65+
if ok {
66+
args := append([]string{"pip", "install"}, extras...)
67+
testutil.RunCommand(t, "uv", args...)
68+
}
69+
70+
projectName := "project_name_" + uniqueProjectId
71+
72+
initConfig := map[string]string{
73+
"project_name": projectName,
74+
"include_notebook": "yes",
75+
"include_python": "yes",
76+
"include_dlt": "yes",
77+
}
78+
b, err := json.Marshal(initConfig)
79+
require.NoError(t, err)
80+
err = os.WriteFile(filepath.Join(tmpDir1, "config.json"), b, 0o644)
81+
require.NoError(t, err)
82+
83+
testcli.RequireOutput(t, ctx, []string{"bundle", "init", "default-python", "--config-file", "config.json"}, "testdata/default_python/bundle_init.txt")
84+
testutil.Chdir(t, projectName)
85+
86+
testcli.RequireOutput(t, ctx, []string{"bundle", "validate"}, "testdata/default_python/bundle_validate.txt")
87+
88+
testcli.RequireOutput(t, ctx, []string{"bundle", "deploy"}, "testdata/default_python/bundle_deploy.txt")
89+
t.Cleanup(func() {
90+
// Delete the stack
91+
testcli.RequireSuccessfulRun(t, ctx, "bundle", "destroy", "--auto-approve")
92+
})
93+
94+
ignoredFields := []string{
95+
"/resources/jobs/project_name_$UNIQUE_PRJ_job/email_notifications",
96+
"/resources/jobs/project_name_$UNIQUE_PRJ_job/job_clusters/0/new_cluster/node_type_id",
97+
"/resources/jobs/project_name_$UNIQUE_PRJ_job/url",
98+
"/resources/pipelines/project_name_$UNIQUE_PRJ_pipeline/catalog",
99+
"/resources/pipelines/project_name_$UNIQUE_PRJ_pipeline/url",
100+
"/workspace/current_user/externalId",
101+
"/workspace/current_user/groups",
102+
"/workspace/current_user/name/familyName",
103+
}
104+
105+
testcli.RequireOutputJQ(t, ctx, []string{"bundle", "summary", "--output", "json"}, "testdata/default_python/bundle_summary.txt", ignoredFields)
106+
}

‎integration/bundle/testdata/default_python/bundle_deploy.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
Building project_name_$UNIQUE_PRJ...
2+
Uploading project_name_$UNIQUE_PRJ-0.0.1+<NUMID>.<NUMID>-py3-none-any.whl...
3+
Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files...
4+
Deploying resources...
5+
Updating deployment state...
6+
Deployment complete!

‎integration/bundle/testdata/default_python/bundle_init.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
Welcome to the default Python template for Databricks Asset Bundles!
3+
Workspace to use (auto-detected, edit in 'project_name_$UNIQUE_PRJ/databricks.yml'): https://$DATABRICKS_HOST
4+
5+
✨ Your new project has been created in the 'project_name_$UNIQUE_PRJ' directory!
6+
7+
Please refer to the README.md file for "getting started" instructions.
8+
See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html.

‎integration/bundle/testdata/default_python/bundle_summary.txt‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
{
2+
"bundle": {
3+
"name": "project_name_$UNIQUE_PRJ",
4+
"target": "dev",
5+
"environment": "dev",
6+
"terraform": {
7+
"exec_path": "/tmp/.../terraform"
8+
},
9+
"git": {
10+
"bundle_root_path": ".",
11+
"inferred": true
12+
},
13+
"mode": "development",
14+
"deployment": {
15+
"lock": {
16+
"enabled": false
17+
}
18+
}
19+
},
20+
"include": [
21+
"resources/project_name_$UNIQUE_PRJ.job.yml",
22+
"resources/project_name_$UNIQUE_PRJ.pipeline.yml"
23+
],
24+
"workspace": {
25+
"host": "https://$DATABRICKS_HOST",
26+
"current_user": {
27+
"active": true,
28+
"displayName": "$USERNAME",
29+
"emails": [
30+
{
31+
"primary": true,
32+
"type": "work",
33+
"value": "$USERNAME"
34+
}
35+
],
36+
"groups": [
37+
{
38+
"$ref": "Groups/$USER.Groups[0]",
39+
"display": "team.engineering",
40+
"type": "direct",
41+
"value": "$USER.Groups[0]"
42+
}
43+
],
44+
"id": "$USER.Id",
45+
"name": {
46+
"familyName": "$USERNAME",
47+
"givenName": "$USERNAME"
48+
},
49+
"schemas": [
50+
"urn:ietf:params:scim:schemas:core:2.0:User",
51+
"urn:ietf:params:scim:schemas:extension:workspace:2.0:User"
52+
],
53+
"short_name": "$USERNAME",
54+
"userName": "$USERNAME"
55+
},
56+
"root_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev",
57+
"file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files",
58+
"resource_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/resources",
59+
"artifact_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/artifacts",
60+
"state_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state"
61+
},
62+
"resources": {
63+
"jobs": {
64+
"project_name_$UNIQUE_PRJ_job": {
65+
"deployment": {
66+
"kind": "BUNDLE",
67+
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
68+
},
69+
"edit_mode": "UI_LOCKED",
70+
"email_notifications": {
71+
"on_failure": [
72+
"$USERNAME"
73+
]
74+
},
75+
"format": "MULTI_TASK",
76+
"id": "<NUMID>",
77+
"job_clusters": [
78+
{
79+
"job_cluster_key": "job_cluster",
80+
"new_cluster": {
81+
"autoscale": {
82+
"max_workers": 4,
83+
"min_workers": 1
84+
},
85+
"node_type_id": "i3.xlarge",
86+
"spark_version": "15.4.x-scala2.12"
87+
}
88+
}
89+
],
90+
"max_concurrent_runs": 4,
91+
"name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_job",
92+
"queue": {
93+
"enabled": true
94+
},
95+
"tags": {
96+
"dev": "$USERNAME"
97+
},
98+
"tasks": [
99+
{
100+
"job_cluster_key": "job_cluster",
101+
"notebook_task": {
102+
"notebook_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/notebook"
103+
},
104+
"task_key": "notebook_task"
105+
},
106+
{
107+
"depends_on": [
108+
{
109+
"task_key": "notebook_task"
110+
}
111+
],
112+
"pipeline_task": {
113+
"pipeline_id": "${resources.pipelines.project_name_$UNIQUE_PRJ_pipeline.id}"
114+
},
115+
"task_key": "refresh_pipeline"
116+
},
117+
{
118+
"depends_on": [
119+
{
120+
"task_key": "refresh_pipeline"
121+
}
122+
],
123+
"job_cluster_key": "job_cluster",
124+
"libraries": [
125+
{
126+
"whl": "dist/*.whl"
127+
}
128+
],
129+
"python_wheel_task": {
130+
"entry_point": "main",
131+
"package_name": "project_name_$UNIQUE_PRJ"
132+
},
133+
"task_key": "main_task"
134+
}
135+
],
136+
"trigger": {
137+
"pause_status": "PAUSED",
138+
"periodic": {
139+
"interval": 1,
140+
"unit": "DAYS"
141+
}
142+
},
143+
"url": "https://$DATABRICKS_HOST/jobs/<NUMID>?o=<NUMID>"
144+
}
145+
},
146+
"pipelines": {
147+
"project_name_$UNIQUE_PRJ_pipeline": {
148+
"catalog": "main",
149+
"configuration": {
150+
"bundle.sourcePath": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src"
151+
},
152+
"deployment": {
153+
"kind": "BUNDLE",
154+
"metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
155+
},
156+
"development": true,
157+
"id": "<UUID>",
158+
"libraries": [
159+
{
160+
"notebook": {
161+
"path": "/Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/dlt_pipeline"
162+
}
163+
}
164+
],
165+
"name": "[dev $USERNAME] project_name_$UNIQUE_PRJ_pipeline",
166+
"target": "project_name_$UNIQUE_PRJ_dev",
167+
"url": "https://$DATABRICKS_HOST/pipelines/<UUID>?o=<NUMID>"
168+
}
169+
}
170+
},
171+
"sync": {
172+
"paths": [
173+
"."
174+
]
175+
},
176+
"presets": {
177+
"name_prefix": "[dev $USERNAME] ",
178+
"pipelines_development": true,
179+
"trigger_pause_status": "PAUSED",
180+
"jobs_max_concurrent_runs": 4,
181+
"tags": {
182+
"dev": "$USERNAME"
183+
}
184+
}
185+
}

‎integration/bundle/testdata/default_python/bundle_validate.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
Name: project_name_$UNIQUE_PRJ
2+
Target: dev
3+
Workspace:
4+
Host: https://$DATABRICKS_HOST
5+
User: $USERNAME
6+
Path: /Workspace/Users/$USERNAME/.bundle/project_name_$UNIQUE_PRJ/dev
7+
8+
Validation OK!

0 commit comments

Comments
 (0)