From 257a8fff56051bd71a3c6042cff026a1ae1dc156 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Dec 2025 11:13:29 +0100 Subject: [PATCH 1/3] direct: pipelines: Ignore run_as change from the backend This matches terraform and works around backend issue where run_as is dropped. --- .../bundle/migrate/runas/databricks.yml.tmpl | 40 +++++++++++++ .../migrate/runas/out.create_requests.json | 0 acceptance/bundle/migrate/runas/out.test.toml | 5 ++ acceptance/bundle/migrate/runas/output.txt | 56 +++++++++++++++++++ acceptance/bundle/migrate/runas/script | 27 +++++++++ acceptance/bundle/migrate/runas/sql.sql | 1 + acceptance/bundle/migrate/runas/test.toml | 3 + bundle/direct/dresources/pipeline.go | 14 ++++- 8 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/migrate/runas/databricks.yml.tmpl create mode 100644 acceptance/bundle/migrate/runas/out.create_requests.json create mode 100644 acceptance/bundle/migrate/runas/out.test.toml create mode 100644 acceptance/bundle/migrate/runas/output.txt create mode 100644 acceptance/bundle/migrate/runas/script create mode 100644 acceptance/bundle/migrate/runas/sql.sql create mode 100644 acceptance/bundle/migrate/runas/test.toml diff --git a/acceptance/bundle/migrate/runas/databricks.yml.tmpl b/acceptance/bundle/migrate/runas/databricks.yml.tmpl new file mode 100644 index 0000000000..02de1d8161 --- /dev/null +++ b/acceptance/bundle/migrate/runas/databricks.yml.tmpl @@ -0,0 +1,40 @@ +bundle: + name: dabs_revenue-$UNIQUE_NAME + +permissions: + - level: CAN_VIEW + group_name: users + +resources: + pipelines: + foo: + name: DABs Revenue Pipeline + + libraries: + - notebook: + path: sql.sql + + serverless: true + +targets: + development: + default: true + mode: development + + resources: + pipelines: + # Make the pipeline write to the current user's schema in logfood. + foo: + catalog: users + target: ${workspace.current_user.short_name} + + production: + run_as: + service_principal_name: $TEST_SP_APPLICATION_ID + + resources: + pipelines: + # Make the pipeline write to the team's schema in logfood. + foo: + catalog: main + target: team_eng_deco diff --git a/acceptance/bundle/migrate/runas/out.create_requests.json b/acceptance/bundle/migrate/runas/out.create_requests.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt new file mode 100644 index 0000000000..e3d03f9251 --- /dev/null +++ b/acceptance/bundle/migrate/runas/output.txt @@ -0,0 +1,56 @@ + +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan -t production +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + +create pipelines.foo +create pipelines.foo.permissions + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy -t production +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //pipelines +File [TEST_TMP_DIR]/out.requests.txt not found + +>>> [CLI] bundle destroy --auto-approve +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + +No active deployment found to destroy! + +Exit code: 1 diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/runas/script new file mode 100644 index 0000000000..887e8e089b --- /dev/null +++ b/acceptance/bundle/migrate/runas/script @@ -0,0 +1,27 @@ +if [ -z "${TEST_SP_APPLICATION_ID+x}" ]; then + export TEST_SP_APPLICATION_ID="123b0db3-ba48-4e12-be3b-3b7898cdf000" +fi + +envsubst < databricks.yml.tmpl > databricks.yml +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan -t production + +cleanup() { + trace $CLI bundle destroy --auto-approve +} + + +trap cleanup EXIT + +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t production +trace print_requests.py //pipelines > out.create_requests.json +trace print_state.py -t production > out.old_state.json + +pipeline_id=$($CLI bundle summary -t production --output json | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get $pipeline_id > out.pipelines_get.json + +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan -t production +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate -t production +trace print_state.py -t production > out.new_state.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -t production +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -t production -o json > out.plan.json +#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment deploy -t production diff --git a/acceptance/bundle/migrate/runas/sql.sql b/acceptance/bundle/migrate/runas/sql.sql new file mode 100644 index 0000000000..ae23affada --- /dev/null +++ b/acceptance/bundle/migrate/runas/sql.sql @@ -0,0 +1 @@ +-- Databricks notebook source diff --git a/acceptance/bundle/migrate/runas/test.toml b/acceptance/bundle/migrate/runas/test.toml new file mode 100644 index 0000000000..254a1c76c9 --- /dev/null +++ b/acceptance/bundle/migrate/runas/test.toml @@ -0,0 +1,3 @@ +Cloud = false +RecordRequests = false +Ignore = [".databricks"] diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 7887192f1c..3919ca1df0 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -120,13 +120,23 @@ func (r *ResourcePipeline) DoDelete(ctx context.Context, id string) error { return r.client.Pipelines.DeleteByPipelineId(ctx, id) } -func (*ResourcePipeline) FieldTriggers(_ bool) map[string]deployplan.ActionType { - return map[string]deployplan.ActionType{ +func (*ResourcePipeline) FieldTriggers(isLocal bool) map[string]deployplan.ActionType { + result := map[string]deployplan.ActionType{ "storage": deployplan.ActionTypeRecreate, "catalog": deployplan.ActionTypeRecreate, "ingestion_definition.connection_name": deployplan.ActionTypeRecreate, "ingestion_definition.ingestion_gateway_id": deployplan.ActionTypeRecreate, } + + if !isLocal { + // We've seen that run_as is not consistently set by the backend + // TF also ignores it (by not copying it here: + // https://github.com/databricks/terraform-provider-databricks/blob/15e951f976e3857d9f01651c4b2513657f137796/pipelines/resource_pipeline.go#L304-L317 + // TODO: Rather than skipping, consider ignoring the change if run_as is not present but still triggering an update if run_as is different. + result["run_as"] = deployplan.ActionTypeSkip + } + + return result } // Note, terraform provider either From bfe343c515e0dca386491ecd4a9b263af0cf9c88 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Dec 2025 11:27:45 +0100 Subject: [PATCH 2/3] clean up comment --- acceptance/bundle/migrate/runas/script | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/runas/script index 887e8e089b..142f910a42 100644 --- a/acceptance/bundle/migrate/runas/script +++ b/acceptance/bundle/migrate/runas/script @@ -24,4 +24,3 @@ trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate -t production trace print_state.py -t production > out.new_state.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -t production trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -t production -o json > out.plan.json -#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment deploy -t production From 33fd9efb6a7ab53a63fd7c337c3851486e86f2f7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 1 Dec 2025 11:38:53 +0100 Subject: [PATCH 3/3] clean up and fix tests --- acceptance/bin/print_state.py | 11 +- .../bundle/migrate/runas/databricks.yml.tmpl | 11 -- .../migrate/runas/out.create_requests.json | 52 ++++++ .../bundle/migrate/runas/out.new_state.json | 47 ++++++ .../bundle/migrate/runas/out.old_state.json | 149 ++++++++++++++++++ .../migrate/runas/out.pipelines_get.json | 28 ++++ acceptance/bundle/migrate/runas/out.plan.json | 64 ++++++++ acceptance/bundle/migrate/runas/output.txt | 103 +++++++++++- acceptance/bundle/migrate/runas/script | 22 +-- acceptance/bundle/migrate/runas/test.toml | 2 +- 10 files changed, 459 insertions(+), 30 deletions(-) create mode 100644 acceptance/bundle/migrate/runas/out.new_state.json create mode 100644 acceptance/bundle/migrate/runas/out.old_state.json create mode 100644 acceptance/bundle/migrate/runas/out.pipelines_get.json create mode 100644 acceptance/bundle/migrate/runas/out.plan.json diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 32f479f1f7..11271b6b4d 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -7,6 +7,7 @@ """ import os +import glob import argparse @@ -19,7 +20,7 @@ def write(filename): def main(): parser = argparse.ArgumentParser() - parser.add_argument("-t", "--target", default="default") + parser.add_argument("-t", "--target") parser.add_argument("--backup", action="store_true") args = parser.parse_args() @@ -27,6 +28,14 @@ def main(): target_dir = f".databricks/bundle/{args.target}" if not os.path.exists(target_dir): raise SystemExit(f"Invalid target {args.target!r}: {target_dir} does not exist") + else: + targets = glob.glob(".databricks/bundle/*") + if not targets: + return + targets = [os.path.basename(x) for x in targets] + if len(targets) > 1: + raise SystemExit("Many targets found, specify one to use with -t: " + ", ".join(sorted(targets))) + args.target = targets[0] if args.backup: filename = f".databricks/bundle/{args.target}/terraform/terraform.tfstate.backup" diff --git a/acceptance/bundle/migrate/runas/databricks.yml.tmpl b/acceptance/bundle/migrate/runas/databricks.yml.tmpl index 02de1d8161..f3d7f72f75 100644 --- a/acceptance/bundle/migrate/runas/databricks.yml.tmpl +++ b/acceptance/bundle/migrate/runas/databricks.yml.tmpl @@ -17,17 +17,6 @@ resources: serverless: true targets: - development: - default: true - mode: development - - resources: - pipelines: - # Make the pipeline write to the current user's schema in logfood. - foo: - catalog: users - target: ${workspace.current_user.short_name} - production: run_as: service_principal_name: $TEST_SP_APPLICATION_ID diff --git a/acceptance/bundle/migrate/runas/out.create_requests.json b/acceptance/bundle/migrate/runas/out.create_requests.json index e69de29bb2..09ff2c23b3 100644 --- a/acceptance/bundle/migrate/runas/out.create_requests.json +++ b/acceptance/bundle/migrate/runas/out.create_requests.json @@ -0,0 +1,52 @@ +{ + "headers": { + "User-Agent": [ + "databricks-tf-provider/1.97.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/pipeline auth/pat" + ] + }, + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql" + } + } + ], + "name": "DABs Revenue Pipeline", + "run_as": { + "service_principal_name": "[UUID]" + }, + "serverless": true, + "target": "team_eng_deco" + } +} +{ + "headers": { + "User-Agent": [ + "databricks-tf-provider/1.97.0 databricks-sdk-go/[SDK_VERSION] go/1.24.0 os/[OS] cli/[DEV_VERSION] terraform/1.5.5 sdk/sdkv2 resource/permissions auth/pat" + ] + }, + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]", + "body": { + "access_control_list": [ + { + "group_name": "users", + "permission_level": "CAN_VIEW" + }, + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json new file mode 100644 index 0000000000..df7d83ab83 --- /dev/null +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -0,0 +1,47 @@ +{ + "lineage": "[UUID]", + "serial": 4, + "state": { + "resources.pipelines.foo": { + "__id__": "[UUID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql" + } + } + ], + "name": "DABs Revenue Pipeline", + "run_as": { + "service_principal_name": "[UUID]" + }, + "serverless": true, + "target": "team_eng_deco" + } + }, + "resources.pipelines.foo.permissions": { + "__id__": "/pipelines/[UUID]", + "state": { + "object_id": "/pipelines/[UUID]", + "permissions": [ + { + "group_name": "users", + "permission_level": "CAN_VIEW" + }, + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } + } +} diff --git a/acceptance/bundle/migrate/runas/out.old_state.json b/acceptance/bundle/migrate/runas/out.old_state.json new file mode 100644 index 0000000000..980c138f8c --- /dev/null +++ b/acceptance/bundle/migrate/runas/out.old_state.json @@ -0,0 +1,149 @@ +{ + "version": 4, + "terraform_version": "1.5.5", + "serial": 3, + "lineage": "[UUID]", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "databricks_permissions", + "name": "pipeline_foo", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "access_control": [ + { + "group_name": "", + "permission_level": "IS_OWNER", + "service_principal_name": "", + "user_name": "[USERNAME]" + }, + { + "group_name": "users", + "permission_level": "CAN_VIEW", + "service_principal_name": "", + "user_name": "" + } + ], + "alert_v2_id": null, + "app_name": null, + "authorization": null, + "cluster_id": null, + "cluster_policy_id": null, + "dashboard_id": null, + "database_instance_name": null, + "directory_id": null, + "directory_path": null, + "experiment_id": null, + "id": "/pipelines/[UUID]", + "instance_pool_id": null, + "job_id": null, + "notebook_id": null, + "notebook_path": null, + "object_type": "pipelines", + "pipeline_id": "[UUID]", + "registered_model_id": null, + "repo_id": null, + "repo_path": null, + "serving_endpoint_id": null, + "sql_alert_id": null, + "sql_dashboard_id": null, + "sql_endpoint_id": null, + "sql_query_id": null, + "vector_search_endpoint_id": null, + "workspace_file_id": null, + "workspace_file_path": null + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "databricks_pipeline.foo" + ] + } + ] + }, + { + "mode": "managed", + "type": "databricks_pipeline", + "name": "foo", + "provider": "provider[\"registry.terraform.io/databricks/databricks\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "allow_duplicate_names": false, + "budget_policy_id": null, + "catalog": "main", + "cause": null, + "channel": "CURRENT", + "cluster": [], + "cluster_id": null, + "configuration": null, + "continuous": false, + "creator_user_name": "[USERNAME]", + "deployment": [ + { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json" + } + ], + "development": false, + "edition": "ADVANCED", + "environment": [], + "event_log": [], + "expected_last_modified": 0, + "filters": [], + "gateway_definition": [], + "health": null, + "id": "[UUID]", + "ingestion_definition": [], + "last_modified": [UNIX_TIME_MILLIS], + "latest_updates": null, + "library": [ + { + "file": [], + "glob": [], + "jar": "", + "maven": [], + "notebook": [ + { + "path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql" + } + ], + "whl": "" + } + ], + "name": "DABs Revenue Pipeline", + "notification": [], + "photon": false, + "restart_window": [], + "root_path": null, + "run_as": [ + { + "service_principal_name": "[UUID]", + "user_name": "" + } + ], + "run_as_user_name": "[USERNAME]", + "schema": null, + "serverless": true, + "state": "IDLE", + "storage": null, + "tags": null, + "target": "team_eng_deco", + "timeouts": null, + "trigger": [], + "url": "[DATABRICKS_URL]/#joblist/pipelines/[UUID]", + "usage_policy_id": null + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjoxMjAwMDAwMDAwMDAwLCJkZWZhdWx0IjoxMjAwMDAwMDAwMDAwLCJkZWxldGUiOjEyMDAwMDAwMDAwMDAsInJlYWQiOjEyMDAwMDAwMDAwMDAsInVwZGF0ZSI6MTIwMDAwMDAwMDAwMH19" + } + ] + } + ], + "check_results": null +} diff --git a/acceptance/bundle/migrate/runas/out.pipelines_get.json b/acceptance/bundle/migrate/runas/out.pipelines_get.json new file mode 100644 index 0000000000..0ad98be90e --- /dev/null +++ b/acceptance/bundle/migrate/runas/out.pipelines_get.json @@ -0,0 +1,28 @@ +{ + "creator_user_name":"[USERNAME]", + "last_modified":[UNIX_TIME_MILLIS], + "name":"DABs Revenue Pipeline", + "pipeline_id":"[UUID]", + "run_as_user_name":"[USERNAME]", + "spec": { + "catalog":"main", + "channel":"CURRENT", + "deployment": { + "kind":"BUNDLE", + "metadata_file_path":"/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json" + }, + "edition":"ADVANCED", + "id":"[UUID]", + "libraries": [ + { + "notebook": { + "path":"/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql" + } + } + ], + "name":"DABs Revenue Pipeline", + "serverless":true, + "target":"team_eng_deco" + }, + "state":"IDLE" +} diff --git a/acceptance/bundle/migrate/runas/out.plan.json b/acceptance/bundle/migrate/runas/out.plan.json new file mode 100644 index 0000000000..404fde7500 --- /dev/null +++ b/acceptance/bundle/migrate/runas/out.plan.json @@ -0,0 +1,64 @@ +{ + "plan": { + "resources.pipelines.foo": { + "action": "skip", + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS], + "name": "DABs Revenue Pipeline", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[UUID]", + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production/files/sql" + } + } + ], + "name": "DABs Revenue Pipeline", + "serverless": true, + "target": "team_eng_deco" + }, + "state": "IDLE" + }, + "changes": { + "remote": { + "run_as": { + "action": "skip" + } + } + } + }, + "resources.pipelines.foo.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.foo", + "label": "${resources.pipelines.foo.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "/pipelines/[UUID]", + "permissions": [ + { + "group_name": "users", + "permission_level": "CAN_VIEW" + }, + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } + } +} diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index e3d03f9251..a118ae1719 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -1,5 +1,5 @@ ->>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan -t production +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. @@ -17,7 +17,7 @@ create pipelines.foo.permissions Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy -t production +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle deploy Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. @@ -36,9 +36,8 @@ Updating deployment state... Deployment complete! >>> print_requests.py //pipelines -File [TEST_TMP_DIR]/out.requests.txt not found ->>> [CLI] bundle destroy --auto-approve +>>> print_state.py Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. @@ -51,6 +50,98 @@ Consider using a adding a top-level permissions section such as the following: See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. in databricks.yml:5:3 -No active deployment found to destroy! -Exit code: 1 +>>> [CLI] pipelines get [UUID] + +>>> DATABRICKS_BUNDLE_ENGINE=terraform [CLI] bundle plan +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle deployment migrate +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + +Migrated 2 resources to direct engine state file: [TEST_TMP_DIR]/.databricks/bundle/production/resources.json + +Validate the migration by running "bundle plan", there should be no actions planned. + +The state file is not synchronized to the workspace yet. To do that and finalize the migration, run "bundle deploy". + +To undo the migration, remove [TEST_TMP_DIR]/.databricks/bundle/production/resources.json and rename [TEST_TMP_DIR]/.databricks/bundle/production/terraform/terraform.tfstate.backup to [TEST_TMP_DIR]/.databricks/bundle/production/terraform/terraform.tfstate + + +>>> print_state.py + +>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan -o json +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + + +>>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle destroy --auto-approve +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:5:3 + +The following resources will be deleted: + delete resources.pipelines.foo + +This action will result in the deletion of the following Lakeflow Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them: + delete resources.pipelines.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/runas/script index 142f910a42..f3675f4d9b 100644 --- a/acceptance/bundle/migrate/runas/script +++ b/acceptance/bundle/migrate/runas/script @@ -3,24 +3,24 @@ if [ -z "${TEST_SP_APPLICATION_ID+x}" ]; then fi envsubst < databricks.yml.tmpl > databricks.yml -trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan -t production +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan cleanup() { - trace $CLI bundle destroy --auto-approve + trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle destroy --auto-approve + rm -f out.requests.txt } - trap cleanup EXIT -trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t production +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy trace print_requests.py //pipelines > out.create_requests.json -trace print_state.py -t production > out.old_state.json +trace print_state.py > out.old_state.json -pipeline_id=$($CLI bundle summary -t production --output json | jq -r '.resources.pipelines.foo.id') +pipeline_id=$(DATABRICKS_BUNDLE_ENGINE= $CLI bundle summary --output json | jq -r '.resources.pipelines.foo.id') trace $CLI pipelines get $pipeline_id > out.pipelines_get.json -trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan -t production -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate -t production -trace print_state.py -t production > out.new_state.json -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -t production -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -t production -o json > out.plan.json +trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate +trace print_state.py > out.new_state.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/migrate/runas/test.toml b/acceptance/bundle/migrate/runas/test.toml index 254a1c76c9..c2c37f10ee 100644 --- a/acceptance/bundle/migrate/runas/test.toml +++ b/acceptance/bundle/migrate/runas/test.toml @@ -1,3 +1,3 @@ Cloud = false -RecordRequests = false +RecordRequests = true Ignore = [".databricks"]