Log to telemetry redacted $resources references (#3567)

denik · web-flow · commit f3ef15bf217a · 2025-09-09T13:09:46.000Z
## Changes For every $resource.group.key.field1.field2 reference add resref_group_field1_field2 bool metric. If we encounter map key, place "mapkey" in case of actual value. If we encounter error during struct traversal, log what we can with resreferr_ prefix instead. ## Why With remote state lookups added in #3531 we use API / SDK struct schema for accessing remote fields, which is different from Terraform schema. Having telemetry allows to estimate number of customers using $resources references and estimate usage of incompatible references. ## Tests Existing tests expanded to log telemetry. New acceptance test which includes longer path. New helper in tests to log boolean fields from telemetry request. New unit tests.
diff --git a/acceptance/bundle/debuglog/out.stderr.direct-exp.txt b/acceptance/bundle/debuglog/out.stderr.direct-exp.txt
@@ -45,6 +45,7 @@
 10:07:59 Debug: Apply pid=12345 mutator=ConfigureWSFS
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources)
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=LogResourceReferences
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=TranslatePathsDashboards
 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources)
diff --git a/acceptance/bundle/debuglog/out.stderr.terraform.txt b/acceptance/bundle/debuglog/out.stderr.terraform.txt
@@ -45,6 +45,7 @@
 10:07:59 Debug: Apply pid=12345 mutator=ConfigureWSFS
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources)
+10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=LogResourceReferences
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths
 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=TranslatePathsDashboards
 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources)
diff --git a/acceptance/bundle/resource_deps/job_tasks/databricks.yml b/acceptance/bundle/resource_deps/job_tasks/databricks.yml
@@ -0,0 +1,31 @@
+resources:
+  jobs:
+    test_job:
+      name: "My Wheel Job"
+      environments: []
+      tags:
+        my_own_tag_key: my_own_tag_value
+      tasks:
+        - task_key: TestTask
+          python_wheel_task:
+            package_name: "my_test_code"
+            entry_point: "run"
+          environment_key: test_env
+          libraries:
+            - whl: hello.whl
+    foo:
+      name: "My foo Job"
+      environments: []
+      tags:
+        my_other_tag: ${resources.jobs.test_job.tags.my_own_tag_key}
+      tasks:
+        - task_key: TestTask
+          python_wheel_task:
+            package_name: "my_test_code"
+            # for direct, the syntax follows SDK struct:
+            #entry_point: ${resources.jobs.test_job.tasks[0].python_wheel_task.entry_point}
+            # for terraform, the syntax follows TF model:
+            entry_point: ${resources.jobs.test_job.task[0].python_wheel_task[0].entry_point}
+          environment_key: test_env
+          libraries:
+            - whl: hello.whl
diff --git a/acceptance/bundle/resource_deps/job_tasks/out.test.toml b/acceptance/bundle/resource_deps/job_tasks/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
diff --git a/acceptance/bundle/resource_deps/job_tasks/output.txt b/acceptance/bundle/resource_deps/job_tasks/output.txt
@@ -0,0 +1,19 @@
+
+>>> [CLI] bundle deploy
+Uploading hello.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_telemetry_bool_values
+experimental.use_legacy_run_as false
+has_classic_interactive_compute false
+has_classic_job_compute false
+has_serverless_compute true
+presets_name_prefix_is_set false
+python_wheel_wrapper_is_set false
+resref_jobs.tags.* true
+resreferr_jobs.task true
+run_as_set false
+skip_artifact_cleanup false
diff --git a/acceptance/bundle/resource_deps/job_tasks/script b/acceptance/bundle/resource_deps/job_tasks/script
@@ -0,0 +1,4 @@
+touch hello.whl
+trace $CLI bundle deploy
+trace print_telemetry_bool_values
+rm out.requests.txt hello.whl
diff --git a/acceptance/bundle/resource_deps/job_tasks/test.toml b/acceptance/bundle/resource_deps/job_tasks/test.toml
@@ -0,0 +1,4 @@
+RecordRequests = true
+
+# direct fails with Error: schema mismatch for ${resources.jobs.test_job.task[0].python_wheel_task[0].entry_point}: task: field "task" not found in resources.Job
+EnvMatrix.DATABRICKS_CLI_DEPLOYMENT = ["terraform"]  # uses TF-specific syntax task[0].python_wheel_task[0]
diff --git a/acceptance/bundle/resource_deps/missing_map_key_tffix/output.txt b/acceptance/bundle/resource_deps/missing_map_key_tffix/output.txt
@@ -42,3 +42,6 @@
     }
   }
 }
+
+=== Nothing is logged in telemetry because deploy has failed
+>>> print_telemetry_bool_values
diff --git a/acceptance/bundle/resource_deps/missing_map_key_tffix/script b/acceptance/bundle/resource_deps/missing_map_key_tffix/script
@@ -1,3 +1,7 @@
 trace $CLI bundle validate -o json | jq .resources
 musterr $CLI bundle plan &> out.plan.$DATABRICKS_CLI_DEPLOYMENT.txt
 musterr $CLI bundle deploy &> out.deploy.$DATABRICKS_CLI_DEPLOYMENT.txt
+
+title "Nothing is logged in telemetry because deploy has failed"
+trace print_telemetry_bool_values
+rm out.requests.txt
diff --git a/acceptance/bundle/resource_deps/missing_map_key_tffix/test.toml b/acceptance/bundle/resource_deps/missing_map_key_tffix/test.toml
@@ -1,2 +1,2 @@
 Badness = "Uploading files even though bundle is wrong; bundle validate does not detect the issue; direct fails but with wrong error message"
-RecordRequests = false
+RecordRequests = true
diff --git a/acceptance/bundle/resource_deps/resources_var/out.requests.txt b/acceptance/bundle/resource_deps/resources_var/out.requests.txt
diff --git a/acceptance/bundle/resource_deps/resources_var/output.txt b/acceptance/bundle/resource_deps/resources_var/output.txt
@@ -32,3 +32,19 @@
     }
   }
 }
+
+>>> jq -s .[] | select(.path=="/api/2.0/pipelines") | .body.name out.requests.txt
+"[dev [USERNAME]] pipeline for mycatalog.myschema.myname"
+
+>>> print_telemetry_bool_values
+experimental.use_legacy_run_as false
+has_classic_interactive_compute false
+has_classic_job_compute false
+has_serverless_compute false
+presets_name_prefix_is_set true
+python_wheel_wrapper_is_set false
+resref_volumes.catalog_name true
+resref_volumes.name true
+resref_volumes.schema_name true
+run_as_set false
+skip_artifact_cleanup false
diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script
@@ -1,5 +1,5 @@
 trace $CLI bundle validate -t dev -o json | jq .resources
 trace errcode $CLI bundle deploy -t dev &> out.deploy.txt
-trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt &> tmp.requests.txt
+trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt
+trace print_telemetry_bool_values
 rm out.requests.txt
-mv tmp.requests.txt out.requests.txt
diff --git a/acceptance/script.prepare b/acceptance/script.prepare
@@ -93,3 +93,7 @@ envsubst() {
     # when MSYS_NO_PATHCONV is enabled.
     env -u MSYS_NO_PATHCONV envsubst.py
 }
+
+print_telemetry_bool_values() {
+    jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | ( (.entry // .) | (.databricks_cli_log.bundle_deploy_event.experimental.bool_values // []) ) | map("\(.key) \(.value)") | .[]' out.requests.txt | sort
+}
diff --git a/bundle/config/mutator/log_resource_references.go b/bundle/config/mutator/log_resource_references.go
@@ -0,0 +1,140 @@
+package mutator
+
+import (
+	"context"
+	"reflect"
+	"strings"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/cli/libs/structaccess"
+)
+
+// Longest field name:
+// >>> len('hard_deletion_sync_min_interval_in_seconds') == 42
+const maxFieldLength = 42
+
+const maxMetricLength = 200
+
+// logResourceReferences scans resources for ${resources.*} references and logs them.
+func LogResourceReferences() bundle.Mutator {
+	return &logResourceReferences{}
+}
+
+type logResourceReferences struct{}
+
+func (m *logResourceReferences) Name() string {
+	return "LogResourceReferences"
+}
+
+func (m *logResourceReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	used := map[string]struct{}{}
+	resources := b.Config.Value().Get("resources")
+	if !resources.IsValid() {
+		// No resources section, nothing to do
+		return nil
+	}
+
+	_ = dyn.WalkReadOnly(resources, func(path dyn.Path, val dyn.Value) error {
+		ref, ok := dynvar.NewRef(val)
+		if !ok {
+			return nil
+		}
+		for _, r := range ref.References() {
+			// Only track ${resources.*} references.
+			if !strings.HasPrefix(r, "resources.") {
+				continue
+			}
+
+			key := convertReferenceToMetric(ctx, b.Config, r)
+			if key != "" {
+				used[key] = struct{}{}
+			}
+		}
+		return nil
+	})
+
+	maxRefsLogged := 50
+
+	// map iteration is randomized, which works for this case
+	for key := range used {
+		b.Metrics.AddBoolValue(key, true)
+		maxRefsLogged--
+		if maxRefsLogged <= 0 {
+			break
+		}
+	}
+
+	return nil
+}
+
+// convertReferenceToMetric converts a reference like "resources.jobs.foo.id" to
+// a telemetry key like "resref__jobs__id"
+func convertReferenceToMetric(ctx context.Context, cfg any, ref string) string {
+	p, err := dyn.NewPathFromString(ref)
+	if err != nil || len(p) < 3 || p[0].Key() != "resources" {
+		return ""
+	}
+
+	group := truncate(p[1].Key(), maxFieldLength, "")
+	kept := []string{"resref_" + group}
+
+	for i := 3; i < len(p); i++ {
+		c := p[i]
+		if c.Key() != "" {
+			item := c.Key()
+
+			repl, err := censorValue(ctx, cfg, p[:i])
+			if err != nil {
+				kept[0] = "resreferr_" + group
+				break
+			}
+
+			if repl != "" {
+				item = repl
+			}
+
+			item = truncate(item, maxFieldLength, "")
+			kept = append(kept, item)
+			continue
+		}
+	}
+
+	result := strings.Join(kept, ".")
+	return truncate(result, maxMetricLength, "__cut")
+}
+
+func truncate(s string, n int, suffix string) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + suffix
+}
+
+func censorValue(ctx context.Context, v any, path dyn.Path) (string, error) {
+	v, err := structaccess.Get(v, path)
+	if err != nil {
+		log.Infof(ctx, "internal error: path=%s: %s", path, err)
+		return "err", err
+	}
+
+	rv := reflect.ValueOf(v)
+	for rv.IsValid() {
+		switch rv.Kind() {
+		case reflect.Pointer, reflect.Interface:
+			if rv.IsNil() {
+				return "", nil
+			}
+			rv = rv.Elem()
+		default:
+			if rv.Kind() == reflect.Map {
+				return "*", nil
+			}
+			return "", nil
+		}
+	}
+	return "", nil
+}
diff --git a/bundle/config/mutator/log_resource_references_test.go b/bundle/config/mutator/log_resource_references_test.go
@@ -0,0 +1,105 @@
+package mutator
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config"
+	cres "github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConvertReferenceToMetric_Table(t *testing.T) {
+	ctx := context.Background()
+	cfg := config.Root{
+		Resources: config.Resources{
+			Jobs: map[string]*cres.Job{
+				"foo": {
+					JobSettings: jobs.JobSettings{
+						Tasks: []jobs.Task{
+							{TaskKey: "alpha"},
+						},
+					},
+				},
+				"niljob": nil,
+			},
+			Apps: map[string]*cres.App{
+				"app1": {
+					Config: map[string]any{
+						"k": "v",
+					},
+				},
+			},
+		},
+	}
+
+	tests := []struct {
+		name string
+		ref  string
+		want string
+	}{
+		{
+			name: "basic job id",
+			ref:  "resources.jobs.foo.id",
+			want: "resref_jobs.id",
+		},
+		{
+			name: "basic job id with non-ascii key",
+			ref:  "resources.jobs.джоб.id",
+			want: "resreferr_jobs",
+		},
+		{
+			name: "invalid empty",
+			ref:  "",
+			want: "",
+		},
+		{
+			name: "invalid variables",
+			ref:  "variables.foo",
+			want: "",
+		},
+		{
+			name: "invalid short resources",
+			ref:  "resources",
+			want: "",
+		},
+		{
+			name: "invalid short group",
+			ref:  "resources.jobs",
+			want: "",
+		},
+		{
+			name: "mapkey censor on app config",
+			ref:  "resources.apps.app1.config.foo",
+			want: "resref_apps.config.*",
+		},
+		{
+			name: "nil job pointer yields plain id",
+			ref:  "resources.jobs.niljob.id",
+			want: "resref_jobs.id",
+		},
+		{
+			name: "err censor on missing job key",
+			ref:  "resources.jobs.missing.id",
+			want: "resreferr_jobs",
+		},
+		{
+			name: "array index tasks key",
+			ref:  "resources.jobs.foo.tasks[0].task_key",
+			want: "resref_jobs.tasks.task_key",
+		},
+		{
+			name: "array index task key",
+			ref:  "resources.jobs.foo.task[0].task_key",
+			want: "resreferr_jobs.task",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := convertReferenceToMetric(ctx, cfg, tt.ref)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go
@@ -45,6 +45,8 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia
 		// Updates (dynamic): resources.* (strings) (resolves variable references to their actual values)
 		// Resolves variable references in 'resources' using bundle, workspace, and variables prefixes
 		mutator.ResolveVariableReferencesOnlyResources(),
+		// After normal variable resolution, log all ${resources.*} references
+		mutator.LogResourceReferences(),
 		mutator.NormalizePaths(),
 
 		// Translate dashboard paths into paths in the workspace file system

Original file line number	Diff line number	Diff line change
`@@ -42,3 +42,6 @@`
`42`	`42`	`}`
`43`	`43`	`}`
`44`	`44`	`}`
	`45`	`+`
	`46`	`+=== Nothing is logged in telemetry because deploy has failed`
	`47`	`+>>> print_telemetry_bool_values`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`	`1`	`Badness = "Uploading files even though bundle is wrong; bundle validate does not detect the issue; direct fails but with wrong error message"`
`2`		`-RecordRequests = false`
	`2`	`+RecordRequests = true`