split StructVar into StructVar + StructVarJSON

Author: denik

bundle/deployplan/plan.go

@@ -64,12 +64,12 @@ func LoadPlanFromFile(path string) (*Plan, error) {
 }
 
 type PlanEntry struct {
-	ID          string               `json:"id,omitempty"`
-	DependsOn   []DependsOnEntry     `json:"depends_on,omitempty"`
-	Action      string               `json:"action,omitempty"`
-	NewState    *structvar.StructVar `json:"new_state,omitempty"`
-	RemoteState any                  `json:"remote_state,omitempty"`
-	Changes     *Changes             `json:"changes,omitempty"`
+	ID          string                   `json:"id,omitempty"`
+	DependsOn   []DependsOnEntry         `json:"depends_on,omitempty"`
+	Action      string                   `json:"action,omitempty"`
+	NewState    *structvar.StructVarJSON `json:"new_state,omitempty"`
+	RemoteState any                      `json:"remote_state,omitempty"`
+	Changes     *Changes                 `json:"changes,omitempty"`
 }
 
 type DependsOnEntry struct {

bundle/direct/bundle_apply.go

@@ -97,12 +97,19 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 		// We don't keep NewState around for 'skip' nodes
 
 		if at != deployplan.ActionTypeSkip {
-			if !b.resolveReferences(ctx, entry, errorPrefix, false, adapter.StateType()) {
+			if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, false) {
 				return false
 			}
 
-			if len(entry.NewState.Refs) > 0 {
-				logdiag.LogError(ctx, fmt.Errorf("%s: unresolved references: %s", errorPrefix, jsonDump(entry.NewState.Refs)))
+			// Get the cached StructVar to check for unresolved refs and get value
+			sv, ok := b.StructVarCache.Load(resourceKey)
+			if !ok {
+				logdiag.LogError(ctx, fmt.Errorf("%s: internal error: missing cached StructVar", errorPrefix))
+				return false
+			}
+
+			if len(sv.Refs) > 0 {
+				logdiag.LogError(ctx, fmt.Errorf("%s: unresolved references: %s", errorPrefix, jsonDump(sv.Refs)))
 				return false
 			}
 
@@ -113,10 +120,10 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 					logdiag.LogError(ctx, fmt.Errorf("state entry not found for %q", resourceKey))
 					return false
 				}
-				err = b.StateDB.SaveState(resourceKey, dbentry.ID, entry.NewState.GetValue(), entry.DependsOn)
+				err = b.StateDB.SaveState(resourceKey, dbentry.ID, sv.Value, entry.DependsOn)
 			} else {
 				// TODO: redo calcDiff to downgrade planned action if possible (?)
-				err = d.Deploy(ctx, &b.StateDB, entry.NewState.GetValue(), at, entry.Changes)
+				err = d.Deploy(ctx, &b.StateDB, sv.Value, at, entry.Changes)
 			}
 
 			if err != nil {

bundle/direct/bundle_plan.go

@@ -50,9 +50,9 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.
 		return err
 	}
 
-	// Load NewState.Value from json.RawMessage into typed structs.
+	// Eagerly parse all StructVarJSON entries to catch parsing errors early.
 	// When the plan is read from JSON, Value contains raw JSON bytes.
-	// We need to parse them into the correct types for each resource.
+	// We parse them into typed structs and cache them for later use.
 	for resourceKey, entry := range plan.Plan {
 		if entry.NewState == nil || len(entry.NewState.Value) == 0 {
 			continue
@@ -63,9 +63,12 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.
 			return fmt.Errorf("cannot convert plan entry %s: %w", resourceKey, err)
 		}
 
-		if err := entry.NewState.Load(adapter.StateType()); err != nil {
+		sv, err := entry.NewState.ToStructVar(adapter.StateType())
+		if err != nil {
 			return fmt.Errorf("cannot load plan entry %s: %w", resourceKey, err)
 		}
+
+		b.StructVarCache.Store(resourceKey, sv)
 	}
 
 	b.Plan = plan
@@ -153,7 +156,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks
 
 		// Process all references in the resource using Refs map
 		// Refs maps path inside resource to references e.g. "${resources.jobs.foo.id} ${resources.jobs.foo.name}"
-		if !b.resolveReferences(ctx, entry, errorPrefix, true, adapter.StateType()) {
+		if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, true) {
 			return false
 		}
 
@@ -180,7 +183,8 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks
 		// for integers: compare 0 with actual object ID. As long as real object IDs are never 0 we're good.
 		// Once we add non-id fields or add per-field details to "bundle plan", we must read dynamic data and deal with references as first class citizen.
 		// This means distinguishing between 0 that are actually object ids and 0 that are there because typed struct integer cannot contain ${...} string.
-		localDiff, err := structdiff.GetStructDiff(savedState, entry.NewState.GetValue(), adapter.KeyedSlices())
+		sv, _ := b.StructVarCache.Load(resourceKey)
+		localDiff, err := structdiff.GetStructDiff(savedState, sv.Value, adapter.KeyedSlices())
 		if err != nil {
 			logdiag.LogError(ctx, fmt.Errorf("%s: diffing local state: %w", errorPrefix, err))
 			return false
@@ -374,13 +378,19 @@ func (b *DeploymentBundle) LookupReferenceLocal(ctx context.Context, path *struc
 		return nil, fmt.Errorf("internal error: %s: action is %q missing new_state", targetResourceKey, targetEntry.Action)
 	}
 
-	_, isUnresolved := targetEntry.NewState.Refs[fieldPathS]
+	// Get StructVar from cache
+	sv, ok := b.StructVarCache.Load(targetResourceKey)
+	if !ok {
+		return nil, fmt.Errorf("internal error: %s: missing cached StructVar", targetResourceKey)
+	}
+
+	_, isUnresolved := sv.Refs[fieldPathS]
 	if isUnresolved {
 		// The value that is requested is itself a reference; this means it will be resolved after apply
 		return nil, errDelayed
 	}
 
-	localConfig := targetEntry.NewState.GetValue()
+	localConfig := sv.Value
 
 	targetGroup := config.GetResourceTypeFromKey(targetResourceKey)
 	adapter := b.Adapters[targetGroup]
@@ -437,17 +447,28 @@ func (b *DeploymentBundle) LookupReferenceLocal(ctx context.Context, path *struc
 	return nil, errDelayed
 }
 
+// getStructVar returns the cached StructVar for the given resource key.
+// The StructVar must have been eagerly loaded during plan creation or InitForApply.
+func (b *DeploymentBundle) getStructVar(resourceKey string) (*structvar.StructVar, error) {
+	sv, ok := b.StructVarCache.Load(resourceKey)
+	if !ok {
+		return nil, fmt.Errorf("internal error: StructVar not found in cache for %s", resourceKey)
+	}
+	return sv, nil
+}
+
 // resolveReferences processes all references in entry.NewState.Refs.
 // If isLocal is true, uses LookupReferenceLocal (for planning phase).
 // If isLocal is false, uses LookupReferenceRemote (for apply phase).
-func (b *DeploymentBundle) resolveReferences(ctx context.Context, entry *deployplan.PlanEntry, errorPrefix string, isLocal bool, stateType reflect.Type) bool {
-	if err := entry.NewState.Load(stateType); err != nil {
-		logdiag.LogError(ctx, fmt.Errorf("%s: cannot load state: %w", errorPrefix, err))
+func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey string, entry *deployplan.PlanEntry, errorPrefix string, isLocal bool) bool {
+	sv, err := b.getStructVar(resourceKey)
+	if err != nil {
+		logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err))
 		return false
 	}
 
 	var resolved bool
-	for fieldPathStr, refString := range entry.NewState.Refs {
+	for fieldPathStr, refString := range sv.Refs {
 		refs, ok := dynvar.NewRef(dyn.V(refString))
 		if !ok {
 			logdiag.LogError(ctx, fmt.Errorf("%s: cannot parse %q", errorPrefix, refString))
@@ -480,7 +501,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, entry *deployp
 				}
 			}
 
-			err = entry.NewState.ResolveRef(ref, value)
+			err = sv.ResolveRef(ref, value)
 			if err != nil {
 				logdiag.LogError(ctx, fmt.Errorf("%s: cannot update %s with value of %q: %w", errorPrefix, fieldPathStr, ref, err))
 				return false
@@ -489,12 +510,14 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, entry *deployp
 		}
 	}
 
+	// Sync resolved values back to StructVarJSON for serialization
 	if resolved {
-		if err := entry.NewState.Save(); err != nil {
+		if err := sv.SyncToJSON(entry.NewState); err != nil {
 			logdiag.LogError(ctx, fmt.Errorf("%s: cannot save state: %w", errorPrefix, err))
 			return false
 		}
 	}
+
 	return true
 }
 
@@ -574,14 +597,14 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root
 			if err != nil {
 				return nil, err
 			}
-			inputConfig = inputConfigStructVar.GetValue()
+			inputConfig = inputConfigStructVar.Value
 			baseRefs = inputConfigStructVar.Refs
 		} else if strings.HasSuffix(node, ".grants") {
 			inputConfigStructVar, err := dresources.PrepareGrantsInputConfig(inputConfig, node)
 			if err != nil {
 				return nil, err
 			}
-			inputConfig = inputConfigStructVar.GetValue()
+			inputConfig = inputConfigStructVar.Value
 			baseRefs = inputConfigStructVar.Refs
 		}
 
@@ -642,14 +665,23 @@ func (b *DeploymentBundle) makePlan(ctx context.Context, configRoot *config.Root
 			return strings.Compare(a.Label, b.Label)
 		})
 
-		newState, err := structvar.NewStructVar(newStateConfig, refs)
+		newState := &structvar.StructVar{
+			Value: newStateConfig,
+			Refs:  refs,
+		}
+
+		// Store in cache for use during planning phase
+		b.StructVarCache.Store(node, newState)
+
+		// Convert to JSON for serialization in plan
+		newStateJSON, err := newState.ToJSON()
 		if err != nil {
-			return nil, fmt.Errorf("%s: cannot create state: %w", node, err)
+			return nil, fmt.Errorf("%s: cannot serialize state: %w", node, err)
 		}
 
 		e := deployplan.PlanEntry{
 			DependsOn: dependsOn,
-			NewState:  newState,
+			NewState:  newStateJSON,
 		}
 
 		p.Plan[node] = &e

bundle/direct/dresources/grants.go

@@ -94,16 +94,16 @@ func PrepareGrantsInputConfig(inputConfig any, node string) (*structvar.StructVa
 		})
 	}
 
-	return structvar.NewStructVar(
-		&GrantsState{
+	return &structvar.StructVar{
+		Value: &GrantsState{
 			SecurableType: securableType,
 			FullName:      "",
 			Grants:        grants,
 		},
-		map[string]string{
+		Refs: map[string]string{
 			"full_name": "${" + baseNode + ".id}",
 		},
-	)
+	}, nil
 }
 
 type ResourceGrants struct {

bundle/direct/dresources/permissions.go

@@ -66,15 +66,15 @@ func PreparePermissionsInputConfig(inputConfig any, node string) (*structvar.Str
 		objectIdRef = prefix + "${" + baseNode + ".endpoint_id}"
 	}
 
-	return structvar.NewStructVar(
-		&PermissionsState{
+	return &structvar.StructVar{
+		Value: &PermissionsState{
 			ObjectID:    "", // Always a reference, defined in Refs below
 			Permissions: permissions,
 		},
-		map[string]string{
+		Refs: map[string]string{
 			"object_id": objectIdRef,
 		},
-	)
+	}, nil
 }
 
 func (*ResourcePermissions) New(client *databricks.WorkspaceClient) *ResourcePermissions {

bundle/direct/dresources/secret_scope_acls.go

@@ -45,15 +45,15 @@ func PrepareSecretScopeAclsInputConfig(inputConfig []resources.SecretScopePermis
 		acls = append(acls, acl)
 	}
 
-	return structvar.NewStructVar(
-		&SecretScopeAclsState{
+	return &structvar.StructVar{
+		Value: &SecretScopeAclsState{
 			ScopeName: "", // Always a reference, defined in Refs below
 			Acls:      acls,
 		},
-		map[string]string{
+		Refs: map[string]string{
 			"scope_name": "${" + baseNode + ".name}",
 		},
-	)
+	}, nil
 }
 
 func (*ResourceSecretScopeAcls) New(client *databricks.WorkspaceClient) *ResourceSecretScopeAcls {

bundle/direct/pkg.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/bundle/direct/dresources"
 	"github.com/databricks/cli/bundle/direct/dstate"
 	"github.com/databricks/cli/bundle/statemgmt/resourcestate"
+	"github.com/databricks/cli/libs/structs/structvar"
 )
 
 // How many parallel operations (API calls) are allowed
@@ -42,6 +43,7 @@ type DeploymentBundle struct {
 	Adapters         map[string]*dresources.Adapter
 	Plan             *deployplan.Plan
 	RemoteStateCache sync.Map
+	StructVarCache   structvar.Cache
 }
 
 // SetRemoteState updates the remote state with type validation and marks as fresh.

cmd/bundle/deployment/migrate.go

@@ -217,18 +217,20 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va
 		// comes from remote state. If we don't store "etag" in state, we won't detect remote drift, because
 		// local=nil, remote="<some new etag>" which will be classified as "server_side_default".
 
-		for key, planEntry := range plan.Plan {
+		for key := range plan.Plan {
 			etag := etags[key]
 			if etag == "" {
 				continue
 			}
-			err := structaccess.Set(planEntry.NewState.GetValue(), structpath.NewStringKey(nil, "etag"), etag)
+			// Get the cached StructVar created during planning
+			sv, ok := deploymentBundle.StructVarCache.Load(key)
+			if !ok {
+				return fmt.Errorf("failed to get cached state for %s", key)
+			}
+			err := structaccess.Set(sv.Value, structpath.NewStringKey(nil, "etag"), etag)
 			if err != nil {
 				return fmt.Errorf("failed to set etag on %s: %w", key, err)
 			}
-			if err := planEntry.NewState.Save(); err != nil {
-				return fmt.Errorf("failed to save state for %s: %w", key, err)
-			}
 		}
 
 		deploymentBundle.Apply(ctx, b.WorkspaceClient(), &b.Config, plan, direct.MigrateMode(true))