direct: Add RemapState resource method (#3586)

## Changes
New RemapState() method on resource that transform remote state to
config type.

## Why
To be used to detect remote drift.

## Tests
Unit tests.

Author: denik

bundle/terranova/tnresources/adapter.go

@@ -26,12 +26,18 @@ type IResource interface {
 	// Example: func (*ResourceJob) PrepareState(input *resources.Job) *jobs.JobSettings
 	PrepareState(input any) any
 
+	// [Required if type(remoteState) != type(state)] RemapState adapts remote state to local state type.
+	// The adapted remote state will then be compared with newState to detect remote drift.
+	// Adaptation is not necessary (but possible) if types already match.
+	// Example: func (*ResourceJob) RemapState(jobs *jobs.Job) *jobs.JobSettings
+	RemapState(input any) any
+
 	// DoRefresh reads and returns remote state from the backend. The return type defines schema for remote field resolution.
-	// Example: func (r *ResourceJob) DoRefresh(ctx context.Context, id string) (*jobs.Job, error) {
+	// Example: func (r *ResourceJob) DoRefresh(ctx context.Context, id string) (*jobs.Job, error)
 	DoRefresh(ctx context.Context, id string) (remoteState any, e error)
 
 	// DoDelete deletes the resource.
-	// Example: func (r *ResourceJob) DoDelete(ctx context.Context, id string) error {
+	// Example: func (r *ResourceJob) DoDelete(ctx context.Context, id string) error
 	DoDelete(ctx context.Context, id string) error
 
 	// [Optional] FieldTriggers returns actions to trigger when given fields are changed.
@@ -49,11 +55,11 @@ type IResourceNoRefresh interface {
 	// We pass newState as a pointer but it is never nil. Changes to it will be persisted in the state, so should be used carefully.
 
 	// DoCreate creates a new resource from the newState.
-	// Example: func (r *ResourceJob) DoCreate(ctx context.Context, newState *jobs.JobSettings) (string, error) {
+	// Example: func (r *ResourceJob) DoCreate(ctx context.Context, newState *jobs.JobSettings) (string, error)
 	DoCreate(ctx context.Context, newState any) (id string, e error)
 
 	// DoUpdate updates the resource. ID must not change as a result of this operation.
-	// Example: func (r *ResourceJob) DoUpdate(ctx context.Context, id string, newState *jobs.JobSettings) error {
+	// Example: func (r *ResourceJob) DoUpdate(ctx context.Context, id string, newState *jobs.JobSettings) error
 	DoUpdate(ctx context.Context, id string, newState any) error
 
 	// [Optional] DoUpdateWithID performs an update that may result in resource having a new ID
@@ -73,11 +79,11 @@ type IResourceNoRefresh interface {
 // Note, resource implementations don't pick between IResourceNoRefresh and IResourceWithRefresh, they can make independent decision for each of the methods.
 type IResourceWithRefresh interface {
 	// DoCreate creates a new resource from the newState. Returns id of the resource and remote state.
-	// Example: func (r *ResourceVolume) DoCreate(ctx context.Context, newState *catalog.CreateWarehouseRequestContent) (string, *catalog.VolumeInfo, error) {
+	// Example: func (r *ResourceVolume) DoCreate(ctx context.Context, newState *catalog.CreateWarehouseRequestContent) (string, *catalog.VolumeInfo, error)
 	DoCreate(ctx context.Context, newState any) (id string, remoteState any, e error)
 
 	// DoUpdate updates the resource. ID must not change as a result of this operation. Returns remote state.
-	// Example: func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, newState *catalog.CreateSchema) (*catalog.SchemaInfo, error) {
+	// Example: func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, newState *catalog.CreateSchema) (*catalog.SchemaInfo, error)
 	DoUpdate(ctx context.Context, id string, newState any) (remoteState any, e error)
 
 	// Optional: updates that may change ID. Returns new id and remote state when available.
@@ -95,6 +101,7 @@ type IResourceWithRefresh interface {
 type Adapter struct {
 	// Required:
 	prepareState *calladapt.BoundCaller
+	remapState   *calladapt.BoundCaller
 	doRefresh    *calladapt.BoundCaller
 	doDelete     *calladapt.BoundCaller
 	doCreate     *calladapt.BoundCaller
@@ -123,6 +130,7 @@ func NewAdapter(typedNil any, client *databricks.WorkspaceClient) (*Adapter, err
 	impl := outs[0]
 	adapter := &Adapter{
 		prepareState:    nil,
+		remapState:      nil,
 		doRefresh:       nil,
 		doDelete:        nil,
 		doCreate:        nil,
@@ -174,6 +182,12 @@ func (a *Adapter) initMethods(resource any) error {
 		return err
 	}
 
+	// RemapState is optional when remote type already matches state type.
+	a.remapState, err = calladapt.PrepareCall(resource, calladapt.TypeOf[IResource](), "RemapState")
+	if err != nil {
+		return err
+	}
+
 	a.doRefresh, err = prepareCallRequired(resource, "DoRefresh")
 	if err != nil {
 		return err
@@ -251,6 +265,17 @@ func (a *Adapter) validate() error {
 		"DoUpdate newState", a.doUpdate.InTypes[2], stateType,
 	}
 
+	// If RemapState is implemented, validate its signature.
+	// Otherwise require remote type to equal state type so remapping isn't needed.
+	if a.remapState != nil {
+		validations = append(validations,
+			"RemapState input", a.remapState.InTypes[0], remoteType,
+			"RemapState return", a.remapState.OutTypes[0], stateType,
+		)
+	} else if remoteType != stateType {
+		return fmt.Errorf("RemapState method not found and remote type %v must match state type %v", remoteType, stateType)
+	}
+
 	// Check if this is WithRefresh version (returns 3 values: id, remoteState, error)
 	if len(a.doCreate.OutTypes) == 3 {
 		validations = append(validations, "DoCreate remoteState return", a.doCreate.OutTypes[1], remoteType)
@@ -323,6 +348,18 @@ func (a *Adapter) PrepareState(input any) (any, error) {
 	return outs[0], nil
 }
 
+func (a *Adapter) RemapState(remoteState any) (any, error) {
+	if a.remapState == nil {
+		return remoteState, nil
+	}
+
+	outs, err := a.remapState.Call(remoteState)
+	if err != nil {
+		return nil, err
+	}
+	return outs[0], nil
+}
+
 func (a *Adapter) DoRefresh(ctx context.Context, id string) (any, error) {
 	outs, err := a.doRefresh.Call(ctx, id)
 	if err != nil {

bundle/terranova/tnresources/all_test.go

@@ -8,13 +8,16 @@ import (
 
 	"github.com/databricks/cli/bundle/config/resources"
 	"github.com/databricks/cli/bundle/deployplan"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/structs/structaccess"
 	"github.com/databricks/cli/libs/structs/structpath"
 	"github.com/databricks/cli/libs/structs/structwalk"
 	"github.com/databricks/cli/libs/testserver"
 	"github.com/databricks/databricks-sdk-go"
 	"github.com/databricks/databricks-sdk-go/service/apps"
 	"github.com/databricks/databricks-sdk-go/service/catalog"
 	"github.com/databricks/databricks-sdk-go/service/database"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -147,6 +150,31 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W
 		require.Equal(t, remote, remoteStateFromWaitUpdate)
 	}
 
+	remappedState, err := adapter.RemapState(remote)
+	require.NoError(t, err)
+	require.NotNil(t, remappedState)
+
+	require.NoError(t, structwalk.Walk(newState, func(path *structpath.PathNode, val any) {
+		remoteValue, err := structaccess.Get(remappedState, dyn.MustPathFromString(path.DynPath()))
+		if err != nil {
+			t.Errorf("Failed to read %s from remapped remote state %#v", path.DynPath(), remappedState)
+		}
+		if val == nil {
+			// t.Logf("Ignoring %s nil, remoteValue=%#v", path.String(), remoteValue)
+			return
+		}
+		v := reflect.ValueOf(val)
+		if v.IsZero() {
+			// t.Logf("Ignoring %s zero (%#v), remoteValue=%#v", path.String(), val, remoteValue)
+			// testserver can set field to backend-generated value
+			return
+		}
+		// t.Logf("Testing %s v=%#v, remoteValue=%#v", path.String(), val, remoteValue)
+		// We expect fields set explicitly to be preserved by testserver, which is true for all resources as of today.
+		// If not true for your resource, add exception here:
+		assert.Equal(t, val, remoteValue, path.DynPath())
+	}))
+
 	err = adapter.DoDelete(ctx, createdID)
 	require.NoError(t, err)
 

bundle/terranova/tnresources/job.go

@@ -24,6 +24,10 @@ func (*ResourceJob) PrepareState(input *resources.Job) *jobs.JobSettings {
 	return &input.JobSettings
 }
 
+func (*ResourceJob) RemapState(jobs *jobs.Job) *jobs.JobSettings {
+	return jobs.Settings
+}
+
 func (r *ResourceJob) DoRefresh(ctx context.Context, id string) (*jobs.Job, error) {
 	idInt, err := parseJobID(id)
 	if err != nil {

bundle/terranova/tnresources/pipeline.go

@@ -23,6 +23,44 @@ func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *pipelines.Crea
 	return &input.CreatePipeline
 }
 
+func (*ResourcePipeline) RemapState(p *pipelines.GetPipelineResponse) *pipelines.CreatePipeline {
+	spec := p.Spec
+	return &pipelines.CreatePipeline{
+		// TODO: Fields that are not available in GetPipelineResponse (like AllowDuplicateNames) should be added to resource's ignore_remote_changes list so that they never produce a call to action
+		AllowDuplicateNames: false,
+		BudgetPolicyId:      spec.BudgetPolicyId,
+		Catalog:             spec.Catalog,
+		Channel:             spec.Channel,
+		Clusters:            spec.Clusters,
+		Configuration:       spec.Configuration,
+		Continuous:          spec.Continuous,
+		Deployment:          spec.Deployment,
+		Development:         spec.Development,
+		DryRun:              false,
+		Edition:             spec.Edition,
+		Environment:         spec.Environment,
+		EventLog:            spec.EventLog,
+		Filters:             spec.Filters,
+		GatewayDefinition:   spec.GatewayDefinition,
+		Id:                  spec.Id,
+		IngestionDefinition: spec.IngestionDefinition,
+		Libraries:           spec.Libraries,
+		Name:                spec.Name,
+		Notifications:       spec.Notifications,
+		Photon:              spec.Photon,
+		RestartWindow:       spec.RestartWindow,
+		RootPath:            spec.RootPath,
+		RunAs:               p.RunAs,
+		Schema:              spec.Schema,
+		Serverless:          spec.Serverless,
+		Storage:             spec.Storage,
+		Tags:                spec.Tags,
+		Target:              spec.Target,
+		Trigger:             spec.Trigger,
+		ForceSendFields:     filterFields[pipelines.CreatePipeline](spec.ForceSendFields, "AllowDuplicateNames", "DryRun", "RunAs"),
+	}
+}
+
 func (r *ResourcePipeline) DoRefresh(ctx context.Context, id string) (*pipelines.GetPipelineResponse, error) {
 	return r.client.Pipelines.GetByPipelineId(ctx, id)
 }

bundle/terranova/tnresources/schema.go

@@ -22,6 +22,17 @@ func (*ResourceSchema) PrepareState(input *resources.Schema) *catalog.CreateSche
 	return &input.CreateSchema
 }
 
+func (*ResourceSchema) RemapState(info *catalog.SchemaInfo) *catalog.CreateSchema {
+	return &catalog.CreateSchema{
+		CatalogName:     info.CatalogName,
+		Comment:         info.Comment,
+		Name:            info.Name,
+		Properties:      info.Properties,
+		StorageRoot:     info.StorageRoot,
+		ForceSendFields: filterFields[catalog.CreateSchema](info.ForceSendFields),
+	}
+}
+
 func (r *ResourceSchema) DoRefresh(ctx context.Context, id string) (*catalog.SchemaInfo, error) {
 	return r.client.Schemas.GetByFullName(ctx, id)
 }

bundle/terranova/tnresources/sql_warehouse.go

@@ -23,6 +23,25 @@ func (*ResourceSqlWarehouse) PrepareState(input *resources.SqlWarehouse) *sql.Cr
 	return &input.CreateWarehouseRequest
 }
 
+func (*ResourceSqlWarehouse) RemapState(warehouse *sql.GetWarehouseResponse) *sql.CreateWarehouseRequest {
+	return &sql.CreateWarehouseRequest{
+		AutoStopMins:            warehouse.AutoStopMins,
+		Channel:                 warehouse.Channel,
+		ClusterSize:             warehouse.ClusterSize,
+		CreatorName:             warehouse.CreatorName,
+		EnablePhoton:            warehouse.EnablePhoton,
+		EnableServerlessCompute: warehouse.EnableServerlessCompute,
+		InstanceProfileArn:      warehouse.InstanceProfileArn,
+		MaxNumClusters:          warehouse.MaxNumClusters,
+		MinNumClusters:          warehouse.MinNumClusters,
+		Name:                    warehouse.Name,
+		SpotInstancePolicy:      warehouse.SpotInstancePolicy,
+		Tags:                    warehouse.Tags,
+		WarehouseType:           sql.CreateWarehouseRequestWarehouseType(warehouse.WarehouseType),
+		ForceSendFields:         filterFields[sql.CreateWarehouseRequest](warehouse.ForceSendFields),
+	}
+}
+
 // DoRefresh reads the warehouse by id.
 func (r *ResourceSqlWarehouse) DoRefresh(ctx context.Context, id string) (*sql.GetWarehouseResponse, error) {
 	return r.client.Warehouses.GetById(ctx, id)

bundle/terranova/tnresources/volume.go

@@ -24,6 +24,18 @@ func (*ResourceVolume) PrepareState(input *resources.Volume) *catalog.CreateVolu
 	return &input.CreateVolumeRequestContent
 }
 
+func (*ResourceVolume) RemapState(info *catalog.VolumeInfo) *catalog.CreateVolumeRequestContent {
+	return &catalog.CreateVolumeRequestContent{
+		CatalogName:     info.CatalogName,
+		Comment:         info.Comment,
+		Name:            info.Name,
+		SchemaName:      info.SchemaName,
+		StorageLocation: info.StorageLocation,
+		VolumeType:      info.VolumeType,
+		ForceSendFields: filterFields[catalog.CreateVolumeRequestContent](info.ForceSendFields),
+	}
+}
+
 func (r *ResourceVolume) DoRefresh(ctx context.Context, id string) (*catalog.VolumeInfo, error) {
 	return r.client.Volumes.ReadByName(ctx, id)
 }