Skip to content

Commit 6164b4e

Browse files
alexottalexott
authored
[Exporter] Add export of PowerBI tasks in datarbicks_job (#4668)
## Changes <!-- Summary of your changes that are easy to understand --> Also: * added support for table update triggers * improved dependencies handling in DBT tasks * refactored specification of job dependencies * refactored matching of schemas and tables ## Tests <!-- How is this tested? Please see the checklist below and also describe any other relevant tests --> - [x] `make test` run locally - [ ] relevant change in `docs/` folder - [ ] covered with integration tests in `internal/acceptance` - [ ] using Go SDK - [ ] using TF Plugin Framework
1 parent 0964337 commit 6164b4e

File tree

4 files changed

+263
-268
lines changed

4 files changed

+263
-268
lines changed

‎NEXT_CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
* Correctly handle account-level identities when generating the code ([#4650](https://github.com/databricks/terraform-provider-databricks/pull/4650))
2222
* Add export of dashboard tasks in `datarbicks_job` ([#4665](https://github.com/databricks/terraform-provider-databricks/pull/4665))
23+
* Add export of PowerBI tasks in `datarbicks_job` ([#4668](https://github.com/databricks/terraform-provider-databricks/pull/4668))
2324
* Add `Ignore` implementation for `databricks_grants` to fix issue with wrongly generated dependencies ([#4661](https://github.com/databricks/terraform-provider-databricks/pull/4650))
2425

2526
### Internal Changes

‎exporter/impl_jobs.go‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,13 +130,46 @@ func importTask(ic *importContext, task sdk_jobs.Task, jobName, rID string) {
130130
}
131131
}
132132
}
133+
if task.PowerBiTask != nil {
134+
if task.PowerBiTask.WarehouseId != "" {
135+
ic.Emit(&resource{
136+
Resource: "databricks_sql_endpoint",
137+
ID: task.PowerBiTask.WarehouseId,
138+
})
139+
}
140+
if task.PowerBiTask.ConnectionResourceName != "" && ic.currentMetastore != nil {
141+
ic.Emit(&resource{
142+
Resource: "databricks_connection",
143+
ID: ic.currentMetastore.MetastoreId + "|" + task.PowerBiTask.ConnectionResourceName,
144+
})
145+
}
146+
for _, table := range task.PowerBiTask.Tables {
147+
if table.Catalog != "" && table.Schema != "" && table.Name != "" {
148+
ic.Emit(&resource{
149+
Resource: "databricks_sql_table",
150+
ID: table.Catalog + "." + table.Schema + "." + table.Name,
151+
})
152+
}
153+
}
154+
155+
}
133156
if task.DbtTask != nil {
134157
if task.DbtTask.WarehouseId != "" {
135158
ic.Emit(&resource{
136159
Resource: "databricks_sql_endpoint",
137160
ID: task.DbtTask.WarehouseId,
138161
})
139162
}
163+
if task.DbtTask.Catalog != "" && task.DbtTask.Schema != "" {
164+
ic.Emit(&resource{
165+
Resource: "databricks_catalog",
166+
ID: task.DbtTask.Catalog,
167+
})
168+
ic.Emit(&resource{
169+
Resource: "databricks_schema",
170+
ID: task.DbtTask.Catalog + "." + task.DbtTask.Schema,
171+
})
172+
}
140173
if task.DbtTask.Source == "WORKSPACE" {
141174
directory := task.DbtTask.ProjectDirectory
142175
if ic.isInRepoOrGitFolder(directory, true) {
@@ -215,6 +248,14 @@ func importJob(ic *importContext, r *resource) error {
215248
})
216249
}
217250
}
251+
if job.Trigger != nil && job.Trigger.TableUpdate != nil {
252+
for _, table := range job.Trigger.TableUpdate.TableNames {
253+
ic.Emit(&resource{
254+
Resource: "databricks_sql_table",
255+
ID: table,
256+
})
257+
}
258+
}
218259
if job.EmailNotifications != nil {
219260
ic.emitListOfUsers(job.EmailNotifications.OnDurationWarningThresholdExceeded)
220261
ic.emitListOfUsers(job.EmailNotifications.OnFailure)
@@ -357,3 +398,174 @@ func (ic *importContext) emitJobsDestinationNotifications(notifications []sdk_jo
357398
})
358399
}
359400
}
401+
402+
var (
403+
// This is the list of dependencies that are needed for a job. It doesn't include the dependencies for for_each_task
404+
// as it will be added by createJobDependencies function.
405+
baseJobDependencies = []reference{
406+
{Path: "job_cluster.new_cluster.aws_attributes.instance_profile_arn", Resource: "databricks_instance_profile"},
407+
{Path: "job_cluster.new_cluster.init_scripts.dbfs.destination", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
408+
{Path: "job_cluster.new_cluster.init_scripts.volumes.destination", Resource: "databricks_file"},
409+
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination", Resource: "databricks_workspace_file"},
410+
{Path: "job_cluster.new_cluster.driver_instance_pool_id", Resource: "databricks_instance_pool"},
411+
{Path: "job_cluster.new_cluster.instance_pool_id", Resource: "databricks_instance_pool"},
412+
{Path: "job_cluster.new_cluster.policy_id", Resource: "databricks_cluster_policy"},
413+
{Path: "run_as.service_principal_name", Resource: "databricks_service_principal", Match: "application_id"},
414+
{Path: "task.dbt_task.warehouse_id", Resource: "databricks_sql_endpoint"},
415+
{Path: "task.dbt_task.catalog", Resource: "databricks_catalog"},
416+
{Path: "task.dbt_task.schema", Resource: "databricks_schema", Match: "name",
417+
IsValidApproximation: createIsMatchingCatalogAndSchema("catalog", "schema"),
418+
},
419+
{Path: "task.dashboard_task.dashboard_id", Resource: "databricks_dashboard"},
420+
{Path: "task.dashboard_task.warehouse_id", Resource: "databricks_sql_endpoint"},
421+
{Path: "task.power_bi_task.warehouse_id", Resource: "databricks_sql_endpoint"},
422+
{Path: "task.power_bi_task.connection_resource_name", Resource: "databricks_connection", Match: "name"},
423+
{Path: "task.power_bi_task.tables.catalog", Resource: "databricks_catalog"},
424+
{Path: "task.power_bi_task.tables.schema", Resource: "databricks_schema", Match: "name",
425+
IsValidApproximation: createIsMatchingCatalogAndSchema("catalog", "schema"),
426+
},
427+
{Path: "task.power_bi_task.tables.name", Resource: "databricks_sql_table", Match: "name",
428+
IsValidApproximation: createIsMatchingCatalogAndSchemaAndTable("catalog", "schema", "name")},
429+
{Path: "task.existing_cluster_id", Resource: "databricks_cluster"},
430+
{Path: "task.library.egg", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
431+
{Path: "task.library.egg", Resource: "databricks_workspace_file", Match: "workspace_path"},
432+
{Path: "task.library.jar", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
433+
{Path: "task.library.jar", Resource: "databricks_file"},
434+
{Path: "task.library.jar", Resource: "databricks_workspace_file", Match: "workspace_path"},
435+
{Path: "task.library.whl", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
436+
{Path: "task.library.whl", Resource: "databricks_file"},
437+
{Path: "task.library.whl", Resource: "databricks_workspace_file", Match: "workspace_path"},
438+
{Path: "task.library.requirements", Resource: "databricks_file"},
439+
{Path: "task.library.requirements", Resource: "databricks_workspace_file", Match: "workspace_path"},
440+
{Path: "task.new_cluster.aws_attributes.instance_profile_arn", Resource: "databricks_instance_profile"},
441+
{Path: "task.new_cluster.init_scripts.dbfs.destination", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
442+
{Path: "task.new_cluster.init_scripts.volumes.destination", Resource: "databricks_file"},
443+
{Path: "task.new_cluster.init_scripts.workspace.destination", Resource: "databricks_workspace_file"},
444+
{Path: "task.new_cluster.instance_pool_id", Resource: "databricks_instance_pool"},
445+
{Path: "task.new_cluster.driver_instance_pool_id", Resource: "databricks_instance_pool"},
446+
{Path: "task.new_cluster.policy_id", Resource: "databricks_cluster_policy"},
447+
{Path: "task.notebook_task.base_parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
448+
{Path: "task.notebook_task.base_parameters", Resource: "databricks_file"},
449+
{Path: "task.notebook_task.base_parameters", Resource: "databricks_workspace_file", Match: "workspace_path"},
450+
{Path: "task.notebook_task.notebook_path", Resource: "databricks_notebook"},
451+
{Path: "task.notebook_task.notebook_path", Resource: "databricks_notebook", Match: "workspace_path"},
452+
{Path: "task.notebook_task.warehouse_id", Resource: "databricks_sql_endpoint"},
453+
{Path: "task.pipeline_task.pipeline_id", Resource: "databricks_pipeline"},
454+
{Path: "task.python_wheel_task.named_parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
455+
{Path: "task.python_wheel_task.named_parameters", Resource: "databricks_file"},
456+
{Path: "task.python_wheel_task.named_parameters", Resource: "databricks_workspace_file", Match: "workspace_path"},
457+
{Path: "task.python_wheel_task.parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
458+
{Path: "task.python_wheel_task.parameters", Resource: "databricks_workspace_file", Match: "workspace_path"},
459+
{Path: "task.run_job_task.job_id", Resource: "databricks_job"},
460+
{Path: "task.run_job_task.job_parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
461+
{Path: "task.run_job_task.job_parameters", Resource: "databricks_workspace_file", Match: "workspace_path"},
462+
{Path: "task.spark_jar_task.jar_uri", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
463+
{Path: "task.spark_jar_task.parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
464+
{Path: "task.spark_jar_task.parameters", Resource: "databricks_file"},
465+
{Path: "task.spark_jar_task.parameters", Resource: "databricks_workspace_file", Match: "workspace_path"},
466+
{Path: "task.spark_python_task.parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
467+
{Path: "task.spark_python_task.python_file", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
468+
{Path: "task.spark_python_task.python_file", Resource: "databricks_workspace_file", Match: "path"},
469+
{Path: "task.spark_python_task.python_file", Resource: "databricks_workspace_file", Match: "workspace_path"},
470+
{Path: "task.spark_submit_task.parameters", Resource: "databricks_dbfs_file", Match: "dbfs_path"},
471+
{Path: "task.spark_submit_task.parameters", Resource: "databricks_file"},
472+
{Path: "task.spark_submit_task.parameters", Resource: "databricks_workspace_file", Match: "workspace_path"},
473+
{Path: "task.sql_task.file.path", Resource: "databricks_workspace_file", Match: "path"},
474+
{Path: "task.sql_task.file.path", Resource: "databricks_workspace_file", Match: "workspace_path"},
475+
{Path: "task.dbt_task.project_directory", Resource: "databricks_directory", Match: "path"},
476+
{Path: "task.dbt_task.project_directory", Resource: "databricks_directory", Match: "workspace_path"},
477+
{Path: "task.sql_task.alert.alert_id", Resource: "databricks_alert"},
478+
{Path: "task.sql_task.alert.subscriptions.destination_id", Resource: "databricks_notification_destination"},
479+
{Path: "task.sql_task.dashboard.dashboard_id", Resource: "databricks_sql_dashboard"},
480+
{Path: "task.sql_task.query.query_id", Resource: "databricks_query"},
481+
{Path: "task.sql_task.warehouse_id", Resource: "databricks_sql_endpoint"},
482+
{Path: "task.webhook_notifications.on_duration_warning_threshold_exceeded.id",
483+
Resource: "databricks_notification_destination"},
484+
{Path: "task.webhook_notifications.on_failure.id", Resource: "databricks_notification_destination"},
485+
{Path: "task.webhook_notifications.on_start.id", Resource: "databricks_notification_destination"},
486+
{Path: "task.webhook_notifications.on_success.id", Resource: "databricks_notification_destination"},
487+
{Path: "task.webhook_notifications.on_streaming_backlog_exceeded.id", Resource: "databricks_notification_destination"},
488+
{Path: "parameter.default", Resource: "databricks_workspace_file", Match: "workspace_path"},
489+
{Path: "parameter.default", Resource: "databricks_workspace_file", Match: "path"},
490+
{Path: "parameter.default", Resource: "databricks_file", Match: "path"},
491+
{Path: "webhook_notifications.on_duration_warning_threshold_exceeded.id",
492+
Resource: "databricks_notification_destination"},
493+
{Path: "webhook_notifications.on_failure.id", Resource: "databricks_notification_destination"},
494+
{Path: "webhook_notifications.on_start.id", Resource: "databricks_notification_destination"},
495+
{Path: "webhook_notifications.on_success.id", Resource: "databricks_notification_destination"},
496+
{Path: "webhook_notifications.on_streaming_backlog_exceeded.id",
497+
Resource: "databricks_notification_destination"},
498+
{Path: "trigger.table_update.table_names", Resource: "databricks_sql_table"},
499+
{Path: "trigger.file_arrival.url", Resource: "databricks_external_location",
500+
Match: "url", MatchType: MatchLongestPrefix},
501+
{Path: "task.sql_task.alert.subscriptions.user_name",
502+
Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
503+
{Path: "task.email_notifications.on_duration_warning_threshold_exceeded", Resource: "databricks_user",
504+
Match: "user_name", MatchType: MatchCaseInsensitive},
505+
{Path: "task.email_notifications.on_failure", Resource: "databricks_user", Match: "user_name",
506+
MatchType: MatchCaseInsensitive},
507+
{Path: "task.email_notifications.on_start", Resource: "databricks_user", Match: "user_name",
508+
MatchType: MatchCaseInsensitive},
509+
{Path: "task.email_notifications.on_success", Resource: "databricks_user", Match: "user_name",
510+
MatchType: MatchCaseInsensitive},
511+
{Path: "task.email_notifications.on_streaming_backlog_exceeded", Resource: "databricks_user",
512+
Match: "user_name", MatchType: MatchCaseInsensitive},
513+
{Path: "run_as.user_name", Resource: "databricks_user", Match: "user_name", MatchType: MatchCaseInsensitive},
514+
{Path: "email_notifications.on_duration_warning_threshold_exceeded", Resource: "databricks_user",
515+
Match: "user_name", MatchType: MatchCaseInsensitive},
516+
{Path: "email_notifications.on_failure", Resource: "databricks_user",
517+
Match: "user_name", MatchType: MatchCaseInsensitive},
518+
{Path: "email_notifications.on_start", Resource: "databricks_user",
519+
Match: "user_name", MatchType: MatchCaseInsensitive},
520+
{Path: "email_notifications.on_success", Resource: "databricks_user",
521+
Match: "user_name", MatchType: MatchCaseInsensitive},
522+
{Path: "email_notifications.on_streaming_backlog_exceeded", Resource: "databricks_user",
523+
Match: "user_name", MatchType: MatchCaseInsensitive},
524+
{Path: "task.library.whl", Resource: "databricks_repo", Match: "workspace_path",
525+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
526+
{Path: "task.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "workspace_path",
527+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
528+
{Path: "task.new_cluster.init_scripts.workspace.destination", Resource: "databricks_repo", Match: "path",
529+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
530+
{Path: "task.notebook_task.base_parameters", Resource: "databricks_repo", Match: "workspace_path",
531+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
532+
{Path: "task.notebook_task.notebook_path", Resource: "databricks_repo", Match: "path",
533+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
534+
{Path: "task.notebook_task.notebook_path", Resource: "databricks_repo", Match: "workspace_path",
535+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
536+
{Path: "task.python_wheel_task.named_parameters", Resource: "databricks_repo", Match: "workspace_path",
537+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
538+
{Path: "task.python_wheel_task.parameters", Resource: "databricks_repo", Match: "workspace_path",
539+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
540+
{Path: "task.run_job_task.job_parameters", Resource: "databricks_repo", Match: "workspace_path",
541+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
542+
{Path: "task.spark_python_task.python_file", Resource: "databricks_repo", Match: "path",
543+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
544+
{Path: "task.spark_python_task.python_file", Resource: "databricks_repo", Match: "workspace_path",
545+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
546+
{Path: "task.spark_jar_task.parameters", Resource: "databricks_repo", Match: "workspace_path",
547+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
548+
{Path: "task.spark_submit_task.parameters", Resource: "databricks_repo", Match: "workspace_path",
549+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
550+
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination",
551+
Resource: "databricks_repo", Match: "workspace_path",
552+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
553+
{Path: "job_cluster.new_cluster.init_scripts.workspace.destination",
554+
Resource: "databricks_repo", Match: "path"},
555+
{Path: "parameter.default", Resource: "databricks_repo", Match: "workspace_path",
556+
MatchType: MatchPrefix, SearchValueTransformFunc: appendEndingSlashToDirName},
557+
}
558+
)
559+
560+
func createJobDependencies() []reference {
561+
dependencies := make([]reference, 0, 2*len(baseJobDependencies))
562+
for _, dep := range baseJobDependencies {
563+
dependencies = append(dependencies, dep)
564+
if strings.HasPrefix(dep.Path, "task.") {
565+
new_dep := dep
566+
new_dep.Path = "task.for_each_task." + dep.Path
567+
dependencies = append(dependencies, new_dep)
568+
}
569+
}
570+
return dependencies
571+
}

‎exporter/impl_uc.go‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -354,11 +354,17 @@ func shouldOmitWithIsolationMode(ic *importContext, pathString string, as *schem
354354
return shouldOmitForUnityCatalog(ic, pathString, as, d)
355355
}
356356

357-
func createIsMatchingCatalogAndSchema(catalog_name_attr, schema_name_attr string) func(ic *importContext, res *resource, ra *resourceApproximation, origPath string) bool {
357+
func createIsMatchingCatalogAndSchema(catalog_name_attr, schema_name_attr string) func(ic *importContext, res *resource,
358+
ra *resourceApproximation, origPath string) bool {
358359
return func(ic *importContext, res *resource, ra *resourceApproximation, origPath string) bool {
359-
// catalog and schema names for the source resource
360-
res_catalog_name := res.Data.Get(catalog_name_attr).(string)
361-
res_schema_name := res.Data.Get(schema_name_attr).(string)
360+
// catalog and schema names for the source resource. We need to copy the original catalog_name_attr
361+
// to a new variable because we're going to modify it
362+
new_catalog_name_attr := catalog_name_attr
363+
if strings.HasSuffix(origPath, "."+schema_name_attr) {
364+
new_catalog_name_attr = strings.TrimSuffix(origPath, schema_name_attr) + catalog_name_attr
365+
}
366+
res_catalog_name := res.Data.Get(new_catalog_name_attr).(string)
367+
res_schema_name := res.Data.Get(origPath).(string)
362368
// In some cases catalog or schema name could be empty, like, in non-UC DLT pipelines, so we need to skip it
363369
if res_catalog_name == "" || res_schema_name == "" {
364370
return false
@@ -377,6 +383,41 @@ func createIsMatchingCatalogAndSchema(catalog_name_attr, schema_name_attr string
377383
}
378384
}
379385

386+
func createIsMatchingCatalogAndSchemaAndTable(catalog_name_attr, schema_name_attr, table_name_attr string) func(ic *importContext, res *resource,
387+
ra *resourceApproximation, origPath string) bool {
388+
return func(ic *importContext, res *resource, ra *resourceApproximation, origPath string) bool {
389+
// catalog and schema names for the source resource. We need to copy the original catalog_name_attr
390+
// to a new variable because we're going to modify it
391+
new_catalog_name_attr := catalog_name_attr
392+
new_schema_name_attr := schema_name_attr
393+
if strings.HasSuffix(origPath, "."+table_name_attr) {
394+
prefix := strings.TrimSuffix(origPath, table_name_attr)
395+
new_catalog_name_attr = prefix + catalog_name_attr
396+
new_schema_name_attr = prefix + schema_name_attr
397+
}
398+
res_catalog_name := res.Data.Get(new_catalog_name_attr).(string)
399+
res_schema_name := res.Data.Get(new_schema_name_attr).(string)
400+
res_table_name := res.Data.Get(origPath).(string)
401+
// In some cases catalog or schema name could be empty, like, in non-UC DLT pipelines, so we need to skip it
402+
if res_catalog_name == "" || res_schema_name == "" || res_table_name == "" {
403+
return false
404+
}
405+
// catalog and schema names for target resource approximation
406+
ra_catalog_name, cat_found := ra.Get("catalog_name")
407+
ra_schema_name, schema_found := ra.Get("schema_name")
408+
ra_table_name, table_found := ra.Get("name")
409+
if !cat_found || !schema_found || !table_found {
410+
log.Printf("[WARN] Can't find attributes in approximation: %s %s, catalog='%v' (found? %v) schema='%v' (found? %v) table='%v' (found? %v). Resource: %s, catalog='%s', schema='%s', table='%s'",
411+
ra.Type, ra.Name, ra_catalog_name, cat_found, ra_schema_name, schema_found, ra_table_name,
412+
table_found, res.Resource, res_catalog_name, res_schema_name, res_table_name)
413+
return false
414+
}
415+
result := ra_catalog_name.(string) == res_catalog_name && ra_schema_name.(string) == res_schema_name && ra_table_name.(string) == res_table_name
416+
return result
417+
418+
}
419+
}
420+
380421
func (ic *importContext) emitWorkspaceBindings(securableType, securableName string) {
381422
bindings, err := ic.workspaceClient.WorkspaceBindings.GetBindingsAll(ic.Context, catalog.GetBindingsRequest{
382423
SecurableName: securableName,

0 commit comments

Comments
 (0)