Skip to content

Commit d2bb4e5

Browse files
authored
[Exporter] Emit WSFS/UC files from parameters in the jobs (#4318)
## Changes <!-- Summary of your changes that are easy to understand --> The global `parameters` array may contain references to files on Workspace File System (WSFS) or UC Volumes. This PR adds emitting of these files and handling references ## Tests <!-- How is this tested? Please see the checklist below and also describe any other relevant tests --> - [x] `make test` run locally - [ ] relevant change in `docs/` folder - [ ] covered with integration tests in `internal/acceptance` - [ ] using Go SDK - [ ] using TF Plugin Framework
1 parent 2b97994 commit d2bb4e5

File tree

2 files changed

+19
-7
lines changed

2 files changed

+19
-7
lines changed

exporter/importables.go

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -444,16 +444,23 @@ var resourcesMap map[string]importable = map[string]importable{
444444
{Path: "task.sql_task.dashboard.dashboard_id", Resource: "databricks_sql_dashboard"},
445445
{Path: "task.sql_task.query.query_id", Resource: "databricks_query"},
446446
{Path: "task.sql_task.warehouse_id", Resource: "databricks_sql_endpoint"},
447-
{Path: "task.webhook_notifications.on_duration_warning_threshold_exceeded.id", Resource: "databricks_notification_destination"},
447+
{Path: "task.webhook_notifications.on_duration_warning_threshold_exceeded.id",
448+
Resource: "databricks_notification_destination"},
448449
{Path: "task.webhook_notifications.on_failure.id", Resource: "databricks_notification_destination"},
449450
{Path: "task.webhook_notifications.on_start.id", Resource: "databricks_notification_destination"},
450451
{Path: "task.webhook_notifications.on_success.id", Resource: "databricks_notification_destination"},
451452
{Path: "task.webhook_notifications.on_streaming_backlog_exceeded.id", Resource: "databricks_notification_destination"},
453+
{Path: "parameter.default", Resource: "databricks_workspace_file", Match: "workspace_path"},
454+
{Path: "parameter.default", Resource: "databricks_workspace_file", Match: "path"},
455+
{Path: "parameter.default", Resource: "databricks_file", Match: "path"},
452456
{Path: "task.email_notifications.on_duration_warning_threshold_exceeded", Resource: "databricks_user",
453457
Match: "user_name", MatchType: MatchCaseInsensitive},
454-
{Path: "task.email_notifications.on_failure", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
455-
{Path: "task.email_notifications.on_start", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
456-
{Path: "task.email_notifications.on_success", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
458+
{Path: "task.email_notifications.on_failure", Resource: "databricks_user", Match: "user_name",
459+
MatchType: MatchCaseInsensitive},
460+
{Path: "task.email_notifications.on_start", Resource: "databricks_user", Match: "user_name",
461+
MatchType: MatchCaseInsensitive},
462+
{Path: "task.email_notifications.on_success", Resource: "databricks_user", Match: "user_name",
463+
MatchType: MatchCaseInsensitive},
457464
{Path: "task.email_notifications.on_streaming_backlog_exceeded", Resource: "databricks_user",
458465
Match: "user_name", MatchType: MatchCaseInsensitive},
459466
{Path: "run_as.user_name", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
@@ -497,7 +504,8 @@ var resourcesMap map[string]importable = map[string]importable{
497504
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
498505
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "workspace_path",
499506
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
500-
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "path",
507+
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "path"},
508+
{Path: "parameter.default", Resource: "databricks_repo", Match: "workspace_path",
501509
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
502510
},
503511
Import: func(ic *importContext, r *resource) error {
@@ -669,6 +677,10 @@ var resourcesMap map[string]importable = map[string]importable{
669677
ic.emitJobsDestinationNotifications(job.WebhookNotifications.OnStart)
670678
ic.emitJobsDestinationNotifications(job.WebhookNotifications.OnStreamingBacklogExceeded)
671679
}
680+
for _, param := range job.Parameters {
681+
ic.emitIfWsfsFile(param.Default)
682+
ic.emitIfVolumeFile(param.Default)
683+
}
672684

673685
return ic.importLibraries(r.Data, s)
674686
},

exporter/util_workspace.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ func hasWorkspacePrefix(path string) bool {
5151
}
5252

5353
func maybeStripWorkspacePrefix(path string) string {
54-
if hasWorkspacePrefix(path) {
55-
return path[10:]
54+
if strings.HasPrefix(path, "/Workspace/") {
55+
return strings.TrimPrefix(path, "/Workspace")
5656
}
5757
return path
5858
}

0 commit comments

Comments
 (0)