Skip to content

Commit 0289bec

Browse files
ilyakuz-dbilyakuz-db
authored
Handle ${workspace.file_path} references in source-linked deployments (#2046)
## Changes 1. Updates `workspace.file_path` during source-linked deployment to address cases like this https://github.com/databricks/bundle-examples/blob/main/default_python/resources/default_python_pipeline.yml#L13 2. Updates `workspace.file_path` in `metadata.json` 3. Prints warning for users when `workspace.file_path` is explicitly set but deploy is running in source-linked mode ## Tests Unit test
1 parent 185bbd2 commit 0289bec

13 files changed

+301
-161
lines changed

‎bundle/config/mutator/apply_presets.go‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99

1010
"github.com/databricks/cli/bundle"
1111
"github.com/databricks/cli/bundle/config"
12-
"github.com/databricks/cli/libs/dbr"
1312
"github.com/databricks/cli/libs/diag"
1413
"github.com/databricks/cli/libs/dyn"
1514
"github.com/databricks/cli/libs/textutil"
@@ -222,27 +221,6 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos
222221
dashboard.DisplayName = prefix + dashboard.DisplayName
223222
}
224223

225-
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
226-
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
227-
if !isDatabricksWorkspace {
228-
target := b.Config.Bundle.Target
229-
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
230-
diags = diags.Append(
231-
diag.Diagnostic{
232-
Severity: diag.Warning,
233-
Summary: "source-linked deployment is available only in the Databricks Workspace",
234-
Paths: []dyn.Path{
235-
path,
236-
},
237-
Locations: b.Config.GetLocations(path[2:].String()),
238-
},
239-
)
240-
241-
disabled := false
242-
b.Config.Presets.SourceLinkedDeployment = &disabled
243-
}
244-
}
245-
246224
return diags
247225
}
248226

‎bundle/config/mutator/apply_presets_test.go‎

Lines changed: 0 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,12 @@ package mutator_test
22

33
import (
44
"context"
5-
"runtime"
65
"testing"
76

87
"github.com/databricks/cli/bundle"
98
"github.com/databricks/cli/bundle/config"
109
"github.com/databricks/cli/bundle/config/mutator"
1110
"github.com/databricks/cli/bundle/config/resources"
12-
"github.com/databricks/cli/bundle/internal/bundletest"
13-
"github.com/databricks/cli/libs/dbr"
14-
"github.com/databricks/cli/libs/dyn"
1511
"github.com/databricks/databricks-sdk-go/service/catalog"
1612
"github.com/databricks/databricks-sdk-go/service/jobs"
1713
"github.com/stretchr/testify/require"
@@ -398,87 +394,3 @@ func TestApplyPresetsResourceNotDefined(t *testing.T) {
398394
})
399395
}
400396
}
401-
402-
func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
403-
if runtime.GOOS == "windows" {
404-
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
405-
}
406-
407-
testContext := context.Background()
408-
enabled := true
409-
disabled := false
410-
workspacePath := "/Workspace/[email protected]"
411-
412-
tests := []struct {
413-
bundlePath string
414-
ctx context.Context
415-
name string
416-
initialValue *bool
417-
expectedValue *bool
418-
expectedWarning string
419-
}{
420-
{
421-
name: "preset enabled, bundle in Workspace, databricks runtime",
422-
bundlePath: workspacePath,
423-
ctx: dbr.MockRuntime(testContext, true),
424-
initialValue: &enabled,
425-
expectedValue: &enabled,
426-
},
427-
{
428-
name: "preset enabled, bundle not in Workspace, databricks runtime",
429-
bundlePath: "/Users/[email protected]",
430-
ctx: dbr.MockRuntime(testContext, true),
431-
initialValue: &enabled,
432-
expectedValue: &disabled,
433-
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
434-
},
435-
{
436-
name: "preset enabled, bundle in Workspace, not databricks runtime",
437-
bundlePath: workspacePath,
438-
ctx: dbr.MockRuntime(testContext, false),
439-
initialValue: &enabled,
440-
expectedValue: &disabled,
441-
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
442-
},
443-
{
444-
name: "preset disabled, bundle in Workspace, databricks runtime",
445-
bundlePath: workspacePath,
446-
ctx: dbr.MockRuntime(testContext, true),
447-
initialValue: &disabled,
448-
expectedValue: &disabled,
449-
},
450-
{
451-
name: "preset nil, bundle in Workspace, databricks runtime",
452-
bundlePath: workspacePath,
453-
ctx: dbr.MockRuntime(testContext, true),
454-
initialValue: nil,
455-
expectedValue: nil,
456-
},
457-
}
458-
459-
for _, tt := range tests {
460-
t.Run(tt.name, func(t *testing.T) {
461-
b := &bundle.Bundle{
462-
SyncRootPath: tt.bundlePath,
463-
Config: config.Root{
464-
Presets: config.Presets{
465-
SourceLinkedDeployment: tt.initialValue,
466-
},
467-
},
468-
}
469-
470-
bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
471-
diags := bundle.Apply(tt.ctx, b, mutator.ApplyPresets())
472-
if diags.HasError() {
473-
t.Fatalf("unexpected error: %v", diags)
474-
}
475-
476-
if tt.expectedWarning != "" {
477-
require.Equal(t, tt.expectedWarning, diags[0].Summary)
478-
require.NotEmpty(t, diags[0].Locations)
479-
}
480-
481-
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
482-
})
483-
}
484-
}

‎bundle/config/mutator/apply_source_linked_deployment_preset.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
package mutator
2+
3+
import (
4+
"context"
5+
"strings"
6+
7+
"github.com/databricks/cli/bundle"
8+
"github.com/databricks/cli/bundle/config"
9+
"github.com/databricks/cli/libs/dbr"
10+
"github.com/databricks/cli/libs/diag"
11+
"github.com/databricks/cli/libs/dyn"
12+
)
13+
14+
type applySourceLinkedDeploymentPreset struct{}
15+
16+
// Apply source-linked deployment preset
17+
func ApplySourceLinkedDeploymentPreset() *applySourceLinkedDeploymentPreset {
18+
return &applySourceLinkedDeploymentPreset{}
19+
}
20+
21+
func (m *applySourceLinkedDeploymentPreset) Name() string {
22+
return "ApplySourceLinkedDeploymentPreset"
23+
}
24+
25+
func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
26+
if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) {
27+
return nil
28+
}
29+
30+
var diags diag.Diagnostics
31+
isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/")
32+
target := b.Config.Bundle.Target
33+
34+
if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) {
35+
if !isDatabricksWorkspace {
36+
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment"))
37+
diags = diags.Append(
38+
diag.Diagnostic{
39+
Severity: diag.Warning,
40+
Summary: "source-linked deployment is available only in the Databricks Workspace",
41+
Paths: []dyn.Path{
42+
path,
43+
},
44+
Locations: b.Config.GetLocations(path[2:].String()),
45+
},
46+
)
47+
48+
disabled := false
49+
b.Config.Presets.SourceLinkedDeployment = &disabled
50+
return diags
51+
}
52+
}
53+
54+
if isDatabricksWorkspace && b.Config.Bundle.Mode == config.Development {
55+
enabled := true
56+
b.Config.Presets.SourceLinkedDeployment = &enabled
57+
}
58+
59+
if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
60+
path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("workspace"), dyn.Key("file_path"))
61+
62+
diags = diags.Append(
63+
diag.Diagnostic{
64+
Severity: diag.Warning,
65+
Summary: "workspace.file_path setting will be ignored in source-linked deployment mode",
66+
Paths: []dyn.Path{
67+
path[2:],
68+
},
69+
Locations: b.Config.GetLocations(path[2:].String()),
70+
},
71+
)
72+
}
73+
74+
return diags
75+
}

‎bundle/config/mutator/apply_source_linked_deployment_preset_test.go‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package mutator_test
2+
3+
import (
4+
"context"
5+
"runtime"
6+
"testing"
7+
8+
"github.com/databricks/cli/bundle"
9+
"github.com/databricks/cli/bundle/config"
10+
"github.com/databricks/cli/bundle/config/mutator"
11+
"github.com/databricks/cli/bundle/internal/bundletest"
12+
"github.com/databricks/cli/libs/dbr"
13+
"github.com/databricks/cli/libs/dyn"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
func TestApplyPresetsSourceLinkedDeployment(t *testing.T) {
18+
if runtime.GOOS == "windows" {
19+
t.Skip("this test is not applicable on Windows because source-linked mode works only in the Databricks Workspace")
20+
}
21+
22+
testContext := context.Background()
23+
enabled := true
24+
disabled := false
25+
workspacePath := "/Workspace/[email protected]"
26+
27+
tests := []struct {
28+
name string
29+
ctx context.Context
30+
mutateBundle func(b *bundle.Bundle)
31+
initialValue *bool
32+
expectedValue *bool
33+
expectedWarning string
34+
}{
35+
{
36+
name: "preset enabled, bundle in Workspace, databricks runtime",
37+
ctx: dbr.MockRuntime(testContext, true),
38+
initialValue: &enabled,
39+
expectedValue: &enabled,
40+
},
41+
{
42+
name: "preset enabled, bundle not in Workspace, databricks runtime",
43+
ctx: dbr.MockRuntime(testContext, true),
44+
mutateBundle: func(b *bundle.Bundle) {
45+
b.SyncRootPath = "/Users/[email protected]"
46+
},
47+
initialValue: &enabled,
48+
expectedValue: &disabled,
49+
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
50+
},
51+
{
52+
name: "preset enabled, bundle in Workspace, not databricks runtime",
53+
ctx: dbr.MockRuntime(testContext, false),
54+
initialValue: &enabled,
55+
expectedValue: &disabled,
56+
expectedWarning: "source-linked deployment is available only in the Databricks Workspace",
57+
},
58+
{
59+
name: "preset disabled, bundle in Workspace, databricks runtime",
60+
ctx: dbr.MockRuntime(testContext, true),
61+
initialValue: &disabled,
62+
expectedValue: &disabled,
63+
},
64+
{
65+
name: "preset nil, bundle in Workspace, databricks runtime",
66+
ctx: dbr.MockRuntime(testContext, true),
67+
initialValue: nil,
68+
expectedValue: nil,
69+
},
70+
{
71+
name: "preset nil, dev mode true, bundle in Workspace, databricks runtime",
72+
ctx: dbr.MockRuntime(testContext, true),
73+
mutateBundle: func(b *bundle.Bundle) {
74+
b.Config.Bundle.Mode = config.Development
75+
},
76+
initialValue: nil,
77+
expectedValue: &enabled,
78+
},
79+
{
80+
name: "preset enabled, workspace.file_path is defined by user",
81+
ctx: dbr.MockRuntime(testContext, true),
82+
mutateBundle: func(b *bundle.Bundle) {
83+
b.Config.Workspace.FilePath = "file_path"
84+
},
85+
initialValue: &enabled,
86+
expectedValue: &enabled,
87+
expectedWarning: "workspace.file_path setting will be ignored in source-linked deployment mode",
88+
},
89+
}
90+
91+
for _, tt := range tests {
92+
t.Run(tt.name, func(t *testing.T) {
93+
b := &bundle.Bundle{
94+
SyncRootPath: workspacePath,
95+
Config: config.Root{
96+
Presets: config.Presets{
97+
SourceLinkedDeployment: tt.initialValue,
98+
},
99+
},
100+
}
101+
102+
if tt.mutateBundle != nil {
103+
tt.mutateBundle(b)
104+
}
105+
106+
bundletest.SetLocation(b, "presets.source_linked_deployment", []dyn.Location{{File: "databricks.yml"}})
107+
bundletest.SetLocation(b, "workspace.file_path", []dyn.Location{{File: "databricks.yml"}})
108+
109+
diags := bundle.Apply(tt.ctx, b, mutator.ApplySourceLinkedDeploymentPreset())
110+
if diags.HasError() {
111+
t.Fatalf("unexpected error: %v", diags)
112+
}
113+
114+
if tt.expectedWarning != "" {
115+
require.Equal(t, tt.expectedWarning, diags[0].Summary)
116+
require.NotEmpty(t, diags[0].Locations)
117+
}
118+
119+
require.Equal(t, tt.expectedValue, b.Config.Presets.SourceLinkedDeployment)
120+
})
121+
}
122+
}

‎bundle/config/mutator/process_target_mode.go‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66

77
"github.com/databricks/cli/bundle"
88
"github.com/databricks/cli/bundle/config"
9-
"github.com/databricks/cli/libs/dbr"
109
"github.com/databricks/cli/libs/diag"
1110
"github.com/databricks/cli/libs/dyn"
1211
"github.com/databricks/cli/libs/iamutil"
@@ -58,14 +57,6 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) {
5857
t.TriggerPauseStatus = config.Paused
5958
}
6059

61-
if !config.IsExplicitlyDisabled(t.SourceLinkedDeployment) {
62-
isInWorkspace := strings.HasPrefix(b.SyncRootPath, "/Workspace/")
63-
if isInWorkspace && dbr.RunsOnRuntime(ctx) {
64-
enabled := true
65-
t.SourceLinkedDeployment = &enabled
66-
}
67-
}
68-
6960
if !config.IsExplicitlyDisabled(t.PipelinesDevelopment) {
7061
enabled := true
7162
t.PipelinesDevelopment = &enabled

0 commit comments

Comments
 (0)