direct: Resolve ${resources.group.key.id} and order deployment accordingly (#3390)

## Changes
The "${resources.<group>.<key>.id}" references are now substituted by
direct deployment backend. They are used to build DAG both during
planning and apply.

## Tests
New test that check jobs update, pipeline recreation and reference going
into integer field.

All existing tests that were disabled due to lack of `${resources}`
support are enabled.

---------

Co-authored-by: Pieter Noordhuis <[email protected]>

Author: denik

acceptance/bundle/deploy/jobs/double-underscore-keys/out.test.toml

@@ -2,4 +2,4 @@ Local = true
 Cloud = true
 
 [EnvMatrix]
-  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/deploy/jobs/double-underscore-keys/test.toml

@@ -1,8 +1,6 @@
 Local = true
 Cloud = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # (only on Cloud/direct) Error: deploying jobs.foo: Method=Jobs.Create *retries.Err *apierr.APIError StatusCode=400 ErrorCode="INVALID_PARAMETER_VALUE" Message="Missing required field: settings.tasks.task_key."
-
 Ignore = [
     "databricks.yml",
 ]

acceptance/bundle/deploy/pipeline/recreate/out.test.toml

@@ -3,4 +3,4 @@ Cloud = true
 RequiresUnityCatalog = true
 
 [EnvMatrix]
-  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/deploy/pipeline/recreate/script

@@ -11,7 +11,7 @@ title "Assert the pipeline is created"
 PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id')
 trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}"
 
-trace $CLI bundle plan --var="catalog=another_catalog"
+trace $CLI bundle plan --var="catalog=another_catalog" 2>&1 | sort
 
 title "Try to redeploy the bundle, pointing the DLT pipeline to a different UC catalog"
 trace errcode $CLI bundle deploy --force-lock --var="catalog=another_catalog"

acceptance/bundle/deploy/pipeline/recreate/test.toml

@@ -2,8 +2,6 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # ${resources} support
-
 Ignore = [
     "databricks.yml"
 ]

acceptance/bundle/deploy/schema/auto-approve/out.test.toml

@@ -3,4 +3,4 @@ Cloud = true
 RequiresUnityCatalog = true
 
 [EnvMatrix]
-  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/deploy/schema/auto-approve/test.toml

@@ -2,8 +2,6 @@ Local = true
 Cloud = true
 RequiresUnityCatalog = true
 
-EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # requires "${resources}"
-
 Ignore = [
     "databricks.yml",
     "test-file-*.txt",

acceptance/bundle/resource_deps/job_id/databricks.yml

@@ -0,0 +1,10 @@
+resources:
+  jobs:
+    bar:
+      name: job bar
+    foo:
+      name: job foo
+      tasks:
+        - task_key: job_task
+          run_job_task:
+            job_id: ${resources.jobs.bar.id}

acceptance/bundle/resource_deps/job_id/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/resource_deps/job_id/output.txt

@@ -0,0 +1,80 @@
+
+>>> [CLI] bundle plan
+create jobs.bar
+create jobs.foo
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests
+{
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "job bar",
+    "queue": {
+      "enabled": true
+    }
+  },
+  "method": "POST",
+  "path": "/api/2.2/jobs/create"
+}
+{
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "job foo",
+    "queue": {
+      "enabled": true
+    },
+    "tasks": [
+      {
+        "run_job_task": {
+          "job_id": [BAR_ID]
+        },
+        "task_key": "job_task"
+      }
+    ]
+  },
+  "method": "POST",
+  "path": "/api/2.2/jobs/create"
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete job bar
+  delete job foo
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_sorted_requests
+{
+  "body": {
+    "job_id": [BAR_ID]
+  },
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete"
+}
+{
+  "body": {
+    "job_id": [FOO_ID]
+  },
+  "method": "POST",
+  "path": "/api/2.2/jobs/delete"
+}