Add hidden "bundle debug refschema" command (#3555)

denik · web-flow · commit 30849a3ab2b9 · 2025-09-08T14:59:22.000Z
## Changes - New hidden command "bundle ref-schema". - Acceptance test that records full output of it. Requires #3531 ## Why Track bundle schema changes. Unlike jsonschema, this includes not just input yaml but stored state and remote resource types. This will help us notice when something disappears or changes type in the state or remote, as this has potential of breaking existing deployments. It is also in greppable format, could be useful for debugging complex $resources.jobs.foo.tasks[0].something.something else references. ## Tests I also added recording of "bundle debug" output under acceptance/bundle/debug.
diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
@@ -84,7 +84,7 @@ const (
 	EntryPointScript = "script"
 	CleanupScript    = "script.cleanup"
 	PrepareScript    = "script.prepare"
-	MaxFileSize      = 100_000
+	MaxFileSize      = 1_000_000
 	// Filename to save replacements to (used by diff.py)
 	ReplsFile = "repls.json"
 	// Filename for materialized config (used as golden file)
diff --git a/acceptance/bundle/debug/output.txt b/acceptance/bundle/debug/output.txt
@@ -1,7 +1,19 @@
-Name: test-bundle
-Target: default
-Workspace:
-  User: [USERNAME]
-  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+Debug information about bundles
 
-Validation OK!
+Usage:
+  databricks bundle debug [command]
+
+Available Commands:
+  refschema   Dump all relevant fields all bundle resources
+
+Flags:
+  -h, --help   help for debug
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
+
+Use "databricks bundle debug [command] --help" for more information about a command.
diff --git a/acceptance/bundle/debug/script b/acceptance/bundle/debug/script
@@ -1 +1 @@
-$CLI bundle validate --debug 2> out.stderr.$DATABRICKS_CLI_DEPLOYMENT.txt
+$CLI bundle debug
diff --git a/acceptance/bundle/debuglog/databricks.yml b/acceptance/bundle/debuglog/databricks.yml
diff --git a/acceptance/bundle/debuglog/out.stderr.direct-exp.txt b/acceptance/bundle/debuglog/out.stderr.direct-exp.txt
diff --git a/acceptance/bundle/debuglog/out.stderr.terraform.txt b/acceptance/bundle/debuglog/out.stderr.terraform.txt
diff --git a/acceptance/bundle/debuglog/out.test.toml b/acceptance/bundle/debuglog/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/debuglog/output.txt b/acceptance/bundle/debuglog/output.txt
@@ -0,0 +1,7 @@
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Validation OK!
diff --git a/acceptance/bundle/debuglog/script b/acceptance/bundle/debuglog/script
@@ -0,0 +1 @@
+$CLI bundle validate --debug 2> out.stderr.$DATABRICKS_CLI_DEPLOYMENT.txt
diff --git a/acceptance/bundle/debuglog/test.toml b/acceptance/bundle/debuglog/test.toml
diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt
diff --git a/acceptance/bundle/refschema/out.test.toml b/acceptance/bundle/refschema/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/bundle/refschema/output.txt b/acceptance/bundle/refschema/output.txt
@@ -0,0 +1,22 @@
+
+>>> [CLI] bundle debug refschema --help
+Dumps all available fields for each resource type by walking the relevant types. Each line is a path to a field, type and set of tags:
+- INPUT: field is present in bundle config.
+- STATE: field is present in the state of the resource (direct deployment only).
+- REMOTE: field is present in the remote state of the resource (direct deployment only).
+- ALL: shortcut for all three.
+
+Usage:
+  databricks bundle debug refschema [flags]
+
+Flags:
+  -h, --help   help for refschema
+
+Global Flags:
+      --debug            enable debug logging
+  -o, --output type      output type: text or json (default text)
+  -p, --profile string   ~/.databrickscfg profile
+  -t, --target string    bundle target to use (if applicable)
+      --var strings      set values for variables defined in bundle config. Example: --var="foo=bar"
+
+>>> [CLI] bundle debug refschema
diff --git a/acceptance/bundle/refschema/script b/acceptance/bundle/refschema/script
@@ -0,0 +1,2 @@
+trace $CLI bundle debug refschema --help
+trace $CLI bundle debug refschema > out.fields.txt
diff --git a/acceptance/bundle/refschema/test.toml b/acceptance/bundle/refschema/test.toml
@@ -0,0 +1,2 @@
+Cloud = false
+Local = true
diff --git a/cmd/bundle/debug.go b/cmd/bundle/debug.go
@@ -14,5 +14,6 @@ func newDebugCommand() *cobra.Command {
 		Hidden: true,
 	}
 	cmd.AddCommand(debug.NewTerraformCommand())
+	cmd.AddCommand(debug.NewRefSchemaCommand())
 	return cmd
 }
diff --git a/cmd/bundle/debug/refschema.go b/cmd/bundle/debug/refschema.go
@@ -0,0 +1,112 @@
+package debug
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"sort"
+	"strings"
+
+	"github.com/databricks/cli/bundle/terranova/tnresources"
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/structdiff/structpath"
+	"github.com/databricks/cli/libs/structwalk"
+	"github.com/databricks/cli/libs/utils"
+	"github.com/spf13/cobra"
+)
+
+func NewRefSchemaCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "refschema",
+		Short: "Dump all relevant fields all bundle resources",
+		Long: `Dumps all available fields for each resource type by walking the relevant types. Each line is a path to a field, type and set of tags:
+- INPUT: field is present in bundle config.
+- STATE: field is present in the state of the resource (direct deployment only).
+- REMOTE: field is present in the remote state of the resource (direct deployment only).
+- ALL: shortcut for all three.
+`,
+		Args: root.NoArgs,
+	}
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		return dumpRemoteSchemas(cmd.OutOrStdout())
+	}
+
+	return cmd
+}
+
+// dumpRemoteSchemas walks through all supported resources and dumps their schema fields.
+func dumpRemoteSchemas(out io.Writer) error {
+	adapters, err := tnresources.InitAll(nil)
+	if err != nil {
+		return fmt.Errorf("failed to initialize adapters: %w", err)
+	}
+
+	for _, resourceName := range utils.SortedKeys(adapters) {
+		adapter := adapters[resourceName]
+
+		// TODO: fields with bundle: tag has variety of behaviors
+		// id is REMOTE but it shows up in inputType
+		// "url" is remote on some resources
+		// "modified_status" in internal
+
+		// path -> typeLabel -> set of sources
+		pathTypes := make(map[string]map[string]map[string]struct{})
+
+		collect := func(root reflect.Type, source string) error {
+			return structwalk.WalkType(root, func(path *structpath.PathNode, typ reflect.Type) bool {
+				if path == nil {
+					return true
+				}
+				p := path.String()
+				p = strings.TrimPrefix(p, ".")
+				t := strings.ReplaceAll(fmt.Sprint(typ), "interface {}", "any")
+				byType, ok := pathTypes[p]
+				if !ok {
+					byType = make(map[string]map[string]struct{})
+					pathTypes[p] = byType
+				}
+				sources, ok := byType[t]
+				if !ok {
+					sources = make(map[string]struct{})
+					byType[t] = sources
+				}
+				sources[source] = struct{}{}
+				return true
+			})
+		}
+
+		if err := collect(adapter.InputConfigType(), "INPUT"); err != nil {
+			return fmt.Errorf("failed to walk input type for %s: %w", resourceName, err)
+		}
+		if err := collect(adapter.ConfigType(), "STATE"); err != nil {
+			return fmt.Errorf("failed to walk config type for %s: %w", resourceName, err)
+		}
+		if err := collect(adapter.RemoteType(), "REMOTE"); err != nil {
+			return fmt.Errorf("failed to walk remote type for %s: %w", resourceName, err)
+		}
+
+		var lines []string
+		for _, p := range utils.SortedKeys(pathTypes) {
+			byType := pathTypes[p]
+			for _, t := range utils.SortedKeys(byType) {
+				info := formatTags(byType[t])
+				lines = append(lines, fmt.Sprintf("resources.%s.*.%s\t%s\t%s\n", resourceName, p, t, info))
+			}
+		}
+
+		sort.Strings(lines)
+		for _, l := range lines {
+			fmt.Fprint(out, l)
+		}
+	}
+
+	return nil
+}
+
+func formatTags(sources map[string]struct{}) string {
+	if len(sources) == 3 {
+		return "ALL"
+	}
+	return strings.Join(utils.SortedKeys(sources), "\t")
+}

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-$CLI bundle validate --debug 2> out.stderr.$DATABRICKS_CLI_DEPLOYMENT.txt`
	`1`	`+$CLI bundle debug`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+$CLI bundle validate --debug 2> out.stderr.$DATABRICKS_CLI_DEPLOYMENT.txt`