diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json new file mode 100644 index 0000000000..babbba49a8 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json @@ -0,0 +1,295 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "update", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } + }, + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [NUMID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "timeout_seconds": 0, + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "webhook_notifications": {} + } + }, + "changes": { + "local": { + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update" + } + }, + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip", + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS][1], + "name": "[dev [USERNAME]] my_default_python_etl", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "id": "[UUID]", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "storage": "dbfs:/pipelines/[UUID]", + "tags": { + "dev": "[USERNAME]" + } + }, + "state": "IDLE" + }, + "changes": { + "remote": { + "storage": { + "action": "skip", + "reason": "server_side_default" + } + } + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.terraform.json new file mode 100644 index 0000000000..5fec9f2a6f --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.terraform.json @@ -0,0 +1,10 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "action": "update" + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json new file mode 100644 index 0000000000..d98c983571 --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json @@ -0,0 +1,217 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [NUMID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_clusters": [ + { + "job_cluster_key": "job_cluster", + "new_cluster": { + "autoscale": { + "max_workers": 4, + "min_workers": 1 + }, + "data_security_mode": "SINGLE_USER", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 0, + "spark_version": "16.4.x-scala2.12" + } + } + ], + "max_concurrent_runs": 1, + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tasks": [ + { + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "job_cluster_key": "job_cluster", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "timeout_seconds": 0, + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "webhook_notifications": {} + } + }, + "changes": { + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.jobs.sample_job.permissions": { + "depends_on": [ + { + "node": "resources.jobs.sample_job", + "label": "${resources.jobs.sample_job.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "/jobs/[NUMID]", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip", + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS][1], + "name": "my_default_python_etl", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, + "id": "[UUID]", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", + "schema": "prod", + "storage": "dbfs:/pipelines/[UUID]" + }, + "state": "IDLE" + }, + "changes": { + "remote": { + "storage": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.pipelines.my_default_python_etl.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "/pipelines/[UUID]", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json new file mode 100644 index 0000000000..2c172f2bce --- /dev/null +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.terraform.json @@ -0,0 +1,16 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "action": "skip" + }, + "resources.jobs.sample_job.permissions": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl.permissions": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/templates/default-python/classic/output.txt b/acceptance/bundle/templates/default-python/classic/output.txt index 7c72b7c72f..a1a9423c92 100644 --- a/acceptance/bundle/templates/default-python/classic/output.txt +++ b/acceptance/bundle/templates/default-python/classic/output.txt @@ -58,6 +58,15 @@ Deploying resources... Updating deployment state... Deployment complete! +>>> [CLI] bundle plan -t dev +Building python_artifact... +update jobs.sample_job + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan -t dev -o json +Building python_artifact... + >>> [CLI] bundle deploy -t prod Building python_artifact... Uploading dist/my_default_python-0.0.1-py3-none-any.whl... @@ -65,3 +74,10 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/ Deploying resources... Updating deployment state... Deployment complete! + +>>> [CLI] bundle plan -t prod +Building python_artifact... +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle plan -t prod -o json +Building python_artifact... diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 471edff34e..398484e362 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -13,15 +13,20 @@ $CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE rm ../../out.requests.txt trace $CLI bundle deploy -t dev print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -t dev # check if if there is drift +trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -t prod # check if there is drift +trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore rm -r .databricks dist cd ../../ +rm out.requests.txt # Calculate the difference from the serverless template diff.py $TESTDIR/../serverless/output output/ > out.compare-vs-serverless.diff diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json new file mode 100644 index 0000000000..d720f47b24 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json @@ -0,0 +1,166 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [NUMID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "timeout_seconds": 0, + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "webhook_notifications": {} + } + }, + "changes": { + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip", + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS][1], + "name": "[dev [USERNAME]] my_default_python_etl", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "id": "[UUID]", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "serverless": true, + "tags": { + "dev": "[USERNAME]" + } + }, + "state": "IDLE" + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json new file mode 100644 index 0000000000..820de2bca6 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.terraform.json @@ -0,0 +1,10 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json new file mode 100644 index 0000000000..7a59132c1d --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json @@ -0,0 +1,195 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "job_id": [NUMID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "timeout_seconds": 0, + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + }, + "webhook_notifications": {} + } + }, + "changes": { + "remote": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default" + } + } + } + }, + "resources.jobs.sample_job.permissions": { + "depends_on": [ + { + "node": "resources.jobs.sample_job", + "label": "${resources.jobs.sample_job.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "/jobs/[NUMID]", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip", + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS][1], + "name": "my_default_python_etl", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, + "id": "[UUID]", + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", + "schema": "prod", + "serverless": true + }, + "state": "IDLE" + } + }, + "resources.pipelines.my_default_python_etl.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "/pipelines/[UUID]", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json new file mode 100644 index 0000000000..2c172f2bce --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.terraform.json @@ -0,0 +1,16 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "action": "skip" + }, + "resources.jobs.sample_job.permissions": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl": { + "action": "skip" + }, + "resources.pipelines.my_default_python_etl.permissions": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json new file mode 100644 index 0000000000..40b373b968 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json @@ -0,0 +1,134 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "${resources.pipelines.my_default_python_etl.id}" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + }, + "vars": { + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.my_default_python_etl.id}" + } + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "create", + "new_state": { + "value": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "serverless": true, + "tags": { + "dev": "[USERNAME]" + } + } + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.terraform.json new file mode 100644 index 0000000000..876a72a786 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.terraform.json @@ -0,0 +1,10 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "action": "create" + }, + "resources.pipelines.my_default_python_etl": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json new file mode 100644 index 0000000000..784709b9c1 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json @@ -0,0 +1,173 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "${resources.pipelines.my_default_python_etl.id}" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + }, + "vars": { + "tasks[2].pipeline_task.pipeline_id": "${resources.pipelines.my_default_python_etl.id}" + } + } + }, + "resources.jobs.sample_job.permissions": { + "depends_on": [ + { + "node": "resources.jobs.sample_job", + "label": "${resources.jobs.sample_job.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/jobs/${resources.jobs.sample_job.id}" + } + } + }, + "resources.pipelines.my_default_python_etl": { + "action": "create", + "new_state": { + "value": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", + "schema": "prod", + "serverless": true + } + } + }, + "resources.pipelines.my_default_python_etl.permissions": { + "depends_on": [ + { + "node": "resources.pipelines.my_default_python_etl", + "label": "${resources.pipelines.my_default_python_etl.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "permissions": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/pipelines/${resources.pipelines.my_default_python_etl.id}" + } + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.terraform.json b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.terraform.json new file mode 100644 index 0000000000..09ed8a43ef --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.terraform.json @@ -0,0 +1,16 @@ +{ + "plan": { + "resources.jobs.sample_job": { + "action": "create" + }, + "resources.jobs.sample_job.permissions": { + "action": "create" + }, + "resources.pipelines.my_default_python_etl": { + "action": "create" + }, + "resources.pipelines.my_default_python_etl.permissions": { + "action": "create" + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt new file mode 100644 index 0000000000..c5d9ae4c6b --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.direct.txt @@ -0,0 +1,176 @@ +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "serverless": true, + "tags": { + "dev": "[USERNAME]" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/.vscode" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/fixtures" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/tests" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt new file mode 100644 index 0000000000..4dfb549b05 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.dev.terraform.txt @@ -0,0 +1,177 @@ +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "[dev [USERNAME]] my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl", + "schema": "[USERNAME]", + "serverless": true, + "tags": { + "dev": "[USERNAME]" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/.vscode" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/fixtures" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/my_default_python_etl/transformations" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/tests" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "[USERNAME]", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "[USERNAME]" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "PAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt new file mode 100644 index 0000000000..d7a761ae4c --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.direct.txt @@ -0,0 +1,212 @@ +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", + "schema": "prod", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/directories/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt new file mode 100644 index 0000000000..ae401a3c97 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/out.requests.prod.terraform.txt @@ -0,0 +1,213 @@ +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "--editable /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files" + ] + }, + "libraries": [ + { + "glob": { + "include": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations/**" + } + } + ], + "name": "my_default_python_etl", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl", + "schema": "prod", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal", + "recursive": true + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/deploy.lock" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/.vscode" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/fixtures" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/resources" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/my_default_python_etl/transformations" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/mkdirs", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/tests" + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/artifacts/.internal/my_default_python-0.0.1-py3-none-any.whl" + ], + "environment_version": "4" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "sample_job", + "parameters": [ + { + "default": "hive_metastore", + "name": "catalog" + }, + { + "default": "prod", + "name": "schema" + } + ], + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files/src/sample_notebook", + "source": "WORKSPACE" + }, + "task_key": "notebook_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "environment_key": "default", + "python_wheel_task": { + "entry_point": "main", + "package_name": "my_default_python", + "parameters": [ + "--catalog", + "hive_metastore", + "--schema", + "prod" + ] + }, + "task_key": "python_wheel_task" + }, + { + "depends_on": [ + { + "task_key": "notebook_task" + } + ], + "pipeline_task": { + "pipeline_id": "[UUID]" + }, + "task_key": "refresh_pipeline" + } + ], + "trigger": { + "pause_status": "UNPAUSED", + "periodic": { + "interval": 1, + "unit": "DAYS" + } + } + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/directories/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]", + "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/templates/default-python/serverless/output.txt b/acceptance/bundle/templates/default-python/serverless/output.txt index d3b26ddd5b..6cad79ec1d 100644 --- a/acceptance/bundle/templates/default-python/serverless/output.txt +++ b/acceptance/bundle/templates/default-python/serverless/output.txt @@ -31,3 +31,51 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod Validation OK! + +>>> [CLI] bundle plan -t dev +Building python_artifact... +create jobs.sample_job +create pipelines.my_default_python_etl + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle plan -t prod +Building python_artifact... +create jobs.sample_job +create jobs.sample_job.permissions +create pipelines.my_default_python_etl +create pipelines.my_default_python_etl.permissions + +Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged +Building python_artifact... +Building python_artifact... + +>>> [CLI] bundle deploy -t dev +Building python_artifact... +Uploading dist/my_default_python-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -t dev +Building python_artifact... +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle plan -t dev -o json +Building python_artifact... + +>>> [CLI] bundle deploy -t prod +Building python_artifact... +Uploading dist/my_default_python-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -t prod +Building python_artifact... +Plan: 0 to add, 0 to change, 0 to delete, 4 unchanged + +>>> [CLI] bundle plan -t prod -o json +Building python_artifact... diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index d7b047fec8..86272b4a26 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -4,6 +4,26 @@ cd output/my_default_python trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod +trace $CLI bundle plan -t dev +trace $CLI bundle plan -t prod + +$CLI bundle plan -o json -t dev > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json + +rm ../../out.requests.txt +trace $CLI bundle deploy -t dev +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -t dev # check if if there is drift +trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle deploy -t prod +print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -t prod # check if there is drift +trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json + # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore -rm .databricks/.gitignore +rm -fr .databricks dist + +cd ../../ +rm out.requests.txt diff --git a/acceptance/bundle/templates/default-python/serverless/test.toml b/acceptance/bundle/templates/default-python/serverless/test.toml new file mode 100644 index 0000000000..159efe0269 --- /dev/null +++ b/acceptance/bundle/templates/default-python/serverless/test.toml @@ -0,0 +1 @@ +RecordRequests = true