internal: update structpath to accept both dot and bracket syntax for fields and maps (#3640)

## Changes
Simplify internal representation of structpath.PathNode: struct fields
and map keys are represented the same way internally.

When parsing, users can specify either `.field` or `[‘field’]`, both
will be encoded as tagStringKey in PathNode.

When saving PathNode as string, String() selects appropriate encoding
(prefers dot unless it uses reserved characters). DynPath() is removed,
String() returns DynPath()-like value for those values that can be
represented by dot notation.

At the same time, PathNode now accurately represents `foo.*` vs
`foo[*]`, this structure is preserved and not equivalent. This again
improves compatibility with dyn.Path as it represents index and keys
wildcard differently.

structaccess is updated to accept PathNode. Since PathNode only gives
StringKey() now and does not distinguish between map and struct fields,
structaccess.Get essentially supports both syntaxes for both kinds.

structwalk is updated to emit dotStar for map wildcards, making it
compatible with code expecting dyn.Pattern (this is used for required
fields and enum validation).

structwalk.PathNode.String() is rewritten to be iterative rather than
recursive.

PathNode gets new methods: Prefix(N), SkipPrefix(N), AsSlice().

## Why
Dot notation is currently used for maps e.g. "resources.jobs.foo". With
precise map encoding that would look like "resources.jobs['foo']" which
looks unfamiliar and also will not match if someone does string
comparison against "resources.jobs.foo".

Adopting this scheme makes it easier to work with PathNode in dyn.Path
world and in fact makes it drop-in replacement (the syntax is superset).

## Tests
Existing and new unit tests.

Author: denik

acceptance/bundle/refschema/out.fields.txt

@@ -11,6 +11,7 @@ import (
 	"github.com/databricks/cli/libs/dyn/dynvar"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/structs/structaccess"
+	"github.com/databricks/cli/libs/structs/structpath"
 )
 
 // Longest field name:
@@ -115,7 +116,14 @@ func truncate(s string, n int, suffix string) string {
 }
 
 func censorValue(ctx context.Context, v any, path dyn.Path) (string, error) {
-	v, err := structaccess.Get(v, path)
+	pathString := path.String()
+	pathNode, err := structpath.Parse(pathString)
+	if err != nil {
+		log.Warnf(ctx, "internal error: parsing %q: %s", pathString, err)
+		return "err", err
+	}
+
+	v, err = structaccess.Get(v, pathNode)
 	if err != nil {
 		log.Infof(ctx, "internal error: path=%s: %s", path, err)
 		return "err", err

bundle/config/mutator/log_resource_references.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/deployplan"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/structs/structaccess"
 	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/structs/structwalk"
@@ -155,9 +154,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W
 	require.NotNil(t, remappedState)
 
 	require.NoError(t, structwalk.Walk(newState, func(path *structpath.PathNode, val any, field *reflect.StructField) {
-		remoteValue, err := structaccess.Get(remappedState, dyn.MustPathFromString(path.DynPath()))
+		remoteValue, err := structaccess.Get(remappedState, path)
 		if err != nil {
-			t.Errorf("Failed to read %s from remapped remote state %#v", path.DynPath(), remappedState)
+			t.Errorf("Failed to read %s from remapped remote state %#v", path.String(), remappedState)
 		}
 		if val == nil {
 			// t.Logf("Ignoring %s nil, remoteValue=%#v", path.String(), remoteValue)
@@ -172,7 +171,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W
 		// t.Logf("Testing %s v=%#v, remoteValue=%#v", path.String(), val, remoteValue)
 		// We expect fields set explicitly to be preserved by testserver, which is true for all resources as of today.
 		// If not true for your resource, add exception here:
-		assert.Equal(t, val, remoteValue, path.DynPath())
+		assert.Equal(t, val, remoteValue, path.String())
 	}))
 
 	err = adapter.DoDelete(ctx, createdID)

bundle/direct/dresources/all_test.go

@@ -9,13 +9,12 @@ import (
 	"github.com/databricks/cli/bundle/deployplan"
 	"github.com/databricks/cli/bundle/direct/dresources"
 	"github.com/databricks/cli/bundle/direct/dstate"
-	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/cli/libs/structs/structdiff"
 	"github.com/databricks/databricks-sdk-go"
 
-	"github.com/databricks/cli/libs/dyn/dynvar"
 	"github.com/databricks/cli/libs/log"
 	"github.com/databricks/cli/libs/structs/structaccess"
+	"github.com/databricks/cli/libs/structs/structpath"
 )
 
 func (d *DeploymentUnit) Plan(ctx context.Context, client *databricks.WorkspaceClient, db *dstate.DeploymentState, inputConfig any, localOnly, refresh bool) (deployplan.ActionType, error) {
@@ -68,12 +67,12 @@ func (d *DeploymentUnit) refreshRemoteState(ctx context.Context, id string) erro
 var ErrDelayed = errors.New("must be resolved after apply")
 
 func (d *DeploymentUnit) ResolveReferenceLocalOrRemote(ctx context.Context, db *dstate.DeploymentState, reference string, actionType deployplan.ActionType, config any) (any, error) {
-	path, ok := dynvar.PureReferenceToPath(reference)
-	if !ok || len(path) <= 3 || path[0:3].String() != d.ResourceKey {
+	path, ok := structpath.PureReferenceToPath(reference)
+	if !ok || path.Len() <= 3 || path.Prefix(3).String() != d.ResourceKey {
 		return nil, fmt.Errorf("internal error: expected reference to %q, got %q", d.ResourceKey, reference)
 	}
 
-	fieldPath := path[3:]
+	fieldPath := path.SkipPrefix(3)
 
 	if fieldPath.String() == "id" {
 		if actionType.KeepsID() {
@@ -136,12 +135,12 @@ func (d *DeploymentUnit) ResolveReferenceLocalOrRemote(ctx context.Context, db *
 }
 
 func (d *DeploymentUnit) ResolveReferenceRemote(ctx context.Context, db *dstate.DeploymentState, reference string) (any, error) {
-	path, ok := dynvar.PureReferenceToPath(reference)
-	if !ok || len(path) <= 3 || path[0:3].String() != d.ResourceKey {
+	path, ok := structpath.PureReferenceToPath(reference)
+	if !ok || path.Len() <= 3 || path.Prefix(3).String() != d.ResourceKey {
 		return nil, fmt.Errorf("internal error: expected reference to %q, got %q", d.ResourceKey, reference)
 	}
 
-	fieldPath := path[3:]
+	fieldPath := path.SkipPrefix(3)
 
 	// Handle "id" field separately - read from state, not remote state
 	if fieldPath.String() == "id" {
@@ -157,7 +156,7 @@ func (d *DeploymentUnit) ResolveReferenceRemote(ctx context.Context, db *dstate.
 }
 
 // ReadRemoteStateField reads a field from remote state with refresh if needed.
-func (d *DeploymentUnit) ReadRemoteStateField(ctx context.Context, db *dstate.DeploymentState, fieldPath dyn.Path) (any, error) {
+func (d *DeploymentUnit) ReadRemoteStateField(ctx context.Context, db *dstate.DeploymentState, fieldPath *structpath.PathNode) (any, error) {
 	// We have options:
 	// 1) Rely on the cached value; refresh if not cached.
 	// 2) Always refresh, read the value.

bundle/direct/plan.go

@@ -132,8 +132,7 @@ func extractEnumFields(typ reflect.Type) ([]EnumPatternInfo, error) {
 				return true
 			}
 
-			fieldPath := path.DynPath()
-			fieldsByPattern[fieldPath] = enumValues
+			fieldsByPattern[path.String()] = enumValues
 		}
 		return true
 	})

bundle/internal/validation/enum.go

@@ -63,12 +63,12 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) {
 			return true
 		}
 
-		fieldName, ok := path.Field()
+		fieldName, ok := path.StringKey()
 		if !ok {
 			return true
 		}
 
-		parentPath := path.Parent().DynPath()
+		parentPath := path.Parent().String()
 		fieldsByPattern[parentPath] = append(fieldsByPattern[parentPath], fieldName)
 		return true
 	})

bundle/internal/validation/required.go

@@ -52,14 +52,6 @@ func TestGet_ConfigRoot_JobTagsAccess(t *testing.T) {
 	require.Error(t, ValidateByString(reflect.TypeOf(root), "resources.jobs.my_job.tags.env.inner"))
 	require.Error(t, ValidateByString(reflect.TypeOf(root), "resources.jobs.my_job.tags1"))
 
-	// Leading dot is allowed
-	v, err = GetByString(root, ".resources.jobs.my_job.tags.team")
-	require.NoError(t, err)
-	require.Equal(t, "platform", v)
-	require.NoError(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags.team"))
-	require.Error(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags.team.inner"))
-	require.Error(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags1"))
-
 	// Array indexing test (1)
 	v, err = GetByString(root, "resources.jobs.my_job.tasks[0].task_key")
 	require.NoError(t, err)

libs/structs/structaccess/bundle_test.go

@@ -1,87 +1,74 @@
 package structaccess
 
 import (
+	"errors"
 	"fmt"
 	"reflect"
-	"strconv"
 
-	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/structs/structtag"
 )
 
 // GetByString returns the value at the given path inside v.
 // This is a convenience function that parses the path string and calls Get.
-//
-// Path grammar (compatible with dyn path):
-//   - Struct field names and map keys separated by '.' (e.g., connection.id)
-//   - (Note, this prevents maps keys that are not id-like from being referenced, but this general problem with references today.)
-//   - Numeric indices in brackets for arrays/slices (e.g., items[0].name)
-//   - Leading '.' is allowed (e.g., .connection.id)
-//
-// Behavior:
-//   - For structs: a key matches a field by its json tag name (if present and not "-").
-//     Embedded anonymous structs are searched.
-//   - For maps: a key indexes map[string]T (or string alias key types).
-//   - For slices/arrays: an index [N] selects the N-th element.
-//   - Wildcards ("*" or "[*]") are not supported and return an error.
 func GetByString(v any, path string) (any, error) {
 	if path == "" {
 		return v, nil
 	}
 
-	dynPath, err := dyn.NewPathFromString(path)
+	pathNode, err := structpath.Parse(path)
 	if err != nil {
 		return nil, err
 	}
 
-	return Get(v, dynPath)
+	return Get(v, pathNode)
 }
 
 // Get returns the value at the given path inside v.
-func Get(v any, path dyn.Path) (any, error) {
-	if len(path) == 0 {
+// Wildcards ("*" or "[*]") are not supported and return an error.
+func Get(v any, path *structpath.PathNode) (any, error) {
+	if path.IsRoot() {
 		return v, nil
 	}
 
+	// Convert path to slice for easier iteration
+	pathSegments := path.AsSlice()
+
 	cur := reflect.ValueOf(v)
-	prefix := ""
-	for _, c := range path {
+	for _, node := range pathSegments {
 		var ok bool
 		cur, ok = deref(cur)
 		if !ok {
 			// cannot proceed further due to nil encountered at current location
-			return nil, fmt.Errorf("%s: cannot access nil value", prefix)
+			return nil, fmt.Errorf("%s: cannot access nil value", node.Parent().String())
 		}
 
-		if c.Key() != "" {
-			// Key access: struct field (by json tag) or map key.
-			newPrefix := prefix
-			if newPrefix == "" {
-				newPrefix = c.Key()
-			} else {
-				newPrefix = newPrefix + "." + c.Key()
+		if idx, isIndex := node.Index(); isIndex {
+			kind := cur.Kind()
+			if kind != reflect.Slice && kind != reflect.Array {
+				return nil, fmt.Errorf("%s: cannot index %s", node.String(), kind)
 			}
-			nv, err := accessKey(cur, c.Key(), newPrefix)
-			if err != nil {
-				return nil, err
+			if idx < 0 || idx >= cur.Len() {
+				return nil, fmt.Errorf("%s: index out of range, length is %d", node.String(), cur.Len())
 			}
-			cur = nv
-			prefix = newPrefix
+			cur = cur.Index(idx)
 			continue
 		}
 
-		// Index access: slice/array
-		idx := c.Index()
-		newPrefix := prefix + "[" + strconv.Itoa(idx) + "]"
-		kind := cur.Kind()
-		if kind != reflect.Slice && kind != reflect.Array {
-			return nil, fmt.Errorf("%s: cannot index %s", newPrefix, kind)
+		if node.DotStar() || node.BracketStar() {
+			return nil, fmt.Errorf("wildcards not supported: %s", path.String())
 		}
-		if idx < 0 || idx >= cur.Len() {
-			return nil, fmt.Errorf("%s: index out of range, length is %d", newPrefix, cur.Len())
+
+		key, ok := node.StringKey()
+		if !ok {
+			return nil, errors.New("unsupported path node type")
+		}
+
+		nv, err := accessKey(cur, key, node)
+		if err != nil {
+			return nil, err
 		}
-		cur = cur.Index(idx)
-		prefix = newPrefix
+		cur = nv
 	}
 
 	// If the current value is invalid (e.g., omitted due to omitempty), return nil.
@@ -100,12 +87,12 @@ func Get(v any, path dyn.Path) (any, error) {
 
 // accessKey returns the field or map entry value selected by key from v.
 // v must be non-pointer, non-interface reflect.Value.
-func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) {
+func accessKey(v reflect.Value, key string, path *structpath.PathNode) (reflect.Value, error) {
 	switch v.Kind() {
 	case reflect.Struct:
 		fv, sf, owner, ok := findStructFieldByKey(v, key)
 		if !ok {
-			return reflect.Value{}, fmt.Errorf("%s: field %q not found in %s", prefix, key, v.Type())
+			return reflect.Value{}, fmt.Errorf("%s: field %q not found in %s", path.String(), key, v.Type())
 		}
 		// Evaluate ForceSendFields on both the current struct and the declaring owner
 		force := containsForceSendField(v, sf.Name) || containsForceSendField(owner, sf.Name)
@@ -129,19 +116,19 @@ func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) {
 	case reflect.Map:
 		kt := v.Type().Key()
 		if kt.Kind() != reflect.String {
-			return reflect.Value{}, fmt.Errorf("%s: map key must be string, got %s", prefix, kt)
+			return reflect.Value{}, fmt.Errorf("%s: map key must be string, got %s", path.String(), kt)
 		}
 		mk := reflect.ValueOf(key)
 		if kt != mk.Type() {
 			mk = mk.Convert(kt)
 		}
 		mv := v.MapIndex(mk)
 		if !mv.IsValid() {
-			return reflect.Value{}, fmt.Errorf("%s: key %q not found in map", prefix, key)
+			return reflect.Value{}, fmt.Errorf("%s: key %q not found in map", path.String(), key)
 		}
 		return mv, nil
 	default:
-		return reflect.Value{}, fmt.Errorf("%s: cannot access key %q on %s", prefix, key, v.Kind())
+		return reflect.Value{}, fmt.Errorf("%s: cannot access key %q on %s", path.String(), key, v.Kind())
 	}
 }
 

libs/structs/structaccess/get.go

@@ -138,10 +138,15 @@ func runCommonTests(t *testing.T, obj any) {
 			want: "bar",
 		},
 		{
-			name: "leading dot allowed",
-			path: ".connection.id",
+			name: "struct field with bracket notation",
+			path: "['connection']['id']",
 			want: "abc",
 		},
+		{
+			name: "map key with bracket notation",
+			path: "labels['env']",
+			want: "dev",
+		},
 
 		// Regular scalar fields - always return zero values
 		{
@@ -169,7 +174,7 @@ func runCommonTests(t *testing.T, obj any) {
 		{
 			name:   "wildcard not supported",
 			path:   "items[*].id",
-			errFmt: "invalid path: items[*].id",
+			errFmt: "wildcards not supported: items[*].id",
 		},
 		{
 			name:   "missing field",