Resolves $resources as regular variables if set by user (#3370)

## Changes
- Resolve references like `${resources.jobs.foo.name}` like regular
variables if they are present in the config.
- Update hasUpdates only if lookupFn returned a valid value (no test
coverage for this, but makes more sense here).

## Why
- It's consistent with how resolutions work in general. This makes
resolved value visible in bundle validate -o json.
- It simplifies direct deployment backend. Now we only need to implement
state/output fields there.
- It minimizes direct & TF implementation differences when we roll out
direct deployment.

## Tests
New test for these references.

Author: denik

NEXT_CHANGELOG.md

@@ -11,6 +11,7 @@
 ### CLI
 
 ### Bundles
+* Changed logic for resolving `${resources...}` references. Previously this would be done by terraform at deploy time. Now if it references a field that is present in the config, it will be done by DABs during bundle loading ([#3370](https://github.com/databricks/cli/pull/3370))
 * Add support for tagging pipelines ([#3086](https://github.com/databricks/cli/pull/3086))
 
 ### API Changes

acceptance/bundle/variables/resolve-resources-fields/databricks.yml

@@ -0,0 +1,23 @@
+variables:
+  a:
+    default: ${resources.schemas.foo.name}
+  b:
+    default: this is var.b ${var.a}
+
+resources:
+  schemas:
+    foo:
+      catalog_name: main
+      name: ${resources.schemas.bar.name}
+      # non_existing and "id" stay unresolved
+      comment: unresolved? schemas.bar.non_existing:"${resources.schemas.bar.non_existing}"  schemas.bar.id:"${resources.schemas.bar.id}"
+    bar:
+      catalog_name: main
+      # "id" and "url" are output only and not present in the config -- should be left unresolved
+      name: unresolved? apps.baz.id="${resources.apps.baz.id}" apps.baz.url="${resources.apps.baz.url}"
+      # includes direct and indirect references to $resources
+      comment: var.b=${var.b} schemas.bar.name=${resources.schemas.bar.name} schemas.foo.name=${resources.schemas.foo.name}
+  apps:
+    baz:
+      name: myapp
+      source_code_path: .

acceptance/bundle/variables/resolve-resources-fields/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/variables/resolve-resources-fields/output.txt

@@ -0,0 +1,31 @@
+{
+  "a": {
+    "default": "${resources.schemas.foo.name}",
+    "value": "${resources.schemas.foo.name}"
+  },
+  "b": {
+    "default": "this is var.b ${resources.schemas.foo.name}",
+    "value": "this is var.b ${resources.schemas.foo.name}"
+  }
+}
+{
+  "apps": {
+    "baz": {
+      "name": "myapp",
+      "permissions": [],
+      "source_code_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files"
+    }
+  },
+  "schemas": {
+    "bar": {
+      "catalog_name": "main",
+      "comment": "var.b=this is var.b unresolved? apps.baz.id=\"${resources.apps.baz.id}\" apps.baz.url=\"${resources.apps.baz.url}\" schemas.bar.name=unresolved? apps.baz.id=\"${resources.apps.baz.id}\" apps.baz.url=\"${resources.apps.baz.url}\" schemas.foo.name=unresolved? apps.baz.id=\"${resources.apps.baz.id}\" apps.baz.url=\"${resources.apps.baz.url}\"",
+      "name": "unresolved? apps.baz.id=\"${resources.apps.baz.id}\" apps.baz.url=\"${resources.apps.baz.url}\""
+    },
+    "foo": {
+      "catalog_name": "main",
+      "comment": "unresolved? schemas.bar.non_existing:\"${resources.schemas.bar.non_existing}\"  schemas.bar.id:\"${resources.schemas.bar.id}\"",
+      "name": "unresolved? apps.baz.id=\"${resources.apps.baz.id}\" apps.baz.url=\"${resources.apps.baz.url}\""
+    }
+  }
+}

acceptance/bundle/variables/resolve-resources-fields/script

@@ -0,0 +1 @@
+$CLI bundle validate -o json | jq '.variables, .resources'

bundle/config/mutator/resolve_variable_references.go

@@ -41,6 +41,14 @@ var defaultPrefixes = []string{
 	"bundle",
 	"workspace",
 	"variables",
+	"resources",
+}
+
+var optionalResolution = map[string]bool{
+	// Enables different mode for resolution:
+	//  - normalization is not done, if field is not set by user it's missing
+	//  - if field is missing (either it's a valid field but not set or invalid field), it remains unresolved, no error
+	"resources": true,
 }
 
 var artifactPath = dyn.MustPathFromString("artifacts")
@@ -229,8 +237,24 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn
 				// Perform resolution only if the path starts with one of the specified prefixes.
 				for _, prefix := range prefixes {
 					if path.HasPrefix(prefix) {
-						hasUpdates = true
-						return m.lookupFn(normalized, path, b)
+						isOpt := optionalResolution[prefix[0].Key()]
+						var value dyn.Value
+						var err error
+						if isOpt {
+							// We don't want injected zero value when resolving $resources.
+							// We only want entries that are explicitly provided by users, so we're using root not normalized here.
+							value, err = m.lookupFn(root, path, b)
+							if !value.IsValid() {
+								// Not having a value is not an error in this case, it might be resolved at deploy time.
+								// TODO: we still could check whether it's part of the schema or not. If latter, we can reject it right away.
+								// TODO: This might be better done after we got rid of TF.
+								return dyn.InvalidValue, dynvar.ErrSkipResolution
+							}
+						} else {
+							value, err = m.lookupFn(normalized, path, b)
+						}
+						hasUpdates = hasUpdates || (err == nil && value.IsValid())
+						return value, err
 					}
 				}