Explicit copy of structs in tnresources (#3352)

## Changes
- In terranova/tnresources, copy structs field-by-field instead of with
copyViaJSON helper.
- Enable exhaustruct linter on tnresources to ensure that all fields are
listed.

## Why
- With this approach we need to name every field, highlighting unused
ones.
- In case SDK adds new fields to request types, we would be alerted and
would need to add them manually.
- We don't rely on json annotation here, might be useful in the future
if SDK removes that.

Note, we still don't have a check that we use every field _from_ config.
Probably need a custom linter for that.

## Tests
Existing tests.

Manually commented out fields to ensure that linter catches that.

Author: denik

.golangci.yaml

@@ -14,6 +14,7 @@ linters:
     - testifylint
     - unconvert
     - unused
+    - exhaustruct
   settings:
     copyloopvar:
       check-alias: true
@@ -73,6 +74,10 @@ linters:
       - common-false-positives
       - legacy
       - std-error-handling
+    rules:
+      - path-except: bundle/terranova/tnresources
+        linters:
+          - exhaustruct
 issues:
   max-issues-per-linter: 1000
   max-same-issues: 1000

bundle/terranova/tnresources/app.go

@@ -27,8 +27,9 @@ func (r *ResourceApp) Config() any {
 
 func (r *ResourceApp) DoCreate(ctx context.Context) (string, error) {
 	request := apps.CreateAppRequest{
-		App:       r.config,
-		NoCompute: true,
+		App:             r.config,
+		NoCompute:       true,
+		ForceSendFields: nil,
 	}
 	waiter, err := r.client.Apps.Create(ctx, request)
 	if err != nil {

bundle/terranova/tnresources/job.go

@@ -73,11 +73,39 @@ func (r *ResourceJob) WaitAfterUpdate(ctx context.Context) error {
 }
 
 func makeCreateJob(config jobs.JobSettings) (jobs.CreateJob, error) {
-	result := jobs.CreateJob{}
+	result := jobs.CreateJob{
+		AccessControlList:    nil, // Not supported by DABs
+		BudgetPolicyId:       config.BudgetPolicyId,
+		Continuous:           config.Continuous,
+		Deployment:           config.Deployment,
+		Description:          config.Description,
+		EditMode:             config.EditMode,
+		EmailNotifications:   config.EmailNotifications,
+		Environments:         config.Environments,
+		Format:               config.Format,
+		GitSource:            config.GitSource,
+		Health:               config.Health,
+		JobClusters:          config.JobClusters,
+		MaxConcurrentRuns:    config.MaxConcurrentRuns,
+		Name:                 config.Name,
+		NotificationSettings: config.NotificationSettings,
+		Parameters:           config.Parameters,
+		PerformanceTarget:    config.PerformanceTarget,
+		Queue:                config.Queue,
+		RunAs:                config.RunAs,
+		Schedule:             config.Schedule,
+		Tags:                 config.Tags,
+		Tasks:                config.Tasks,
+		TimeoutSeconds:       config.TimeoutSeconds,
+		Trigger:              config.Trigger,
+		WebhookNotifications: config.WebhookNotifications,
+
+		ForceSendFields: filterFields[jobs.CreateJob](config.ForceSendFields),
+	}
+
 	// TODO: Validate copy - all fields must be initialized or explicitly allowed to be empty
 	// Unset AccessControlList
-	err := copyViaJSON(&result, config)
-	return result, err
+	return result, nil
 }
 
 func makeResetJob(config jobs.JobSettings, id string) (jobs.ResetJob, error) {

bundle/terranova/tnresources/pipeline.go

@@ -33,14 +33,42 @@ func (r *ResourcePipeline) DoCreate(ctx context.Context) (string, error) {
 }
 
 func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string) error {
-	request := pipelines.EditPipeline{}
-	err := copyViaJSON(&request, r.config)
-	if err != nil {
-		return err
+	request := pipelines.EditPipeline{
+		AllowDuplicateNames:  r.config.AllowDuplicateNames,
+		BudgetPolicyId:       r.config.BudgetPolicyId,
+		Catalog:              r.config.Catalog,
+		Channel:              r.config.Channel,
+		Clusters:             r.config.Clusters,
+		Configuration:        r.config.Configuration,
+		Continuous:           r.config.Continuous,
+		Deployment:           r.config.Deployment,
+		Development:          r.config.Development,
+		Edition:              r.config.Edition,
+		Environment:          r.config.Environment,
+		EventLog:             r.config.EventLog,
+		ExpectedLastModified: 0,
+		Filters:              r.config.Filters,
+		GatewayDefinition:    r.config.GatewayDefinition,
+		Id:                   r.config.Id,
+		IngestionDefinition:  r.config.IngestionDefinition,
+		Libraries:            r.config.Libraries,
+		Name:                 r.config.Name,
+		Notifications:        r.config.Notifications,
+		Photon:               r.config.Photon,
+		RestartWindow:        r.config.RestartWindow,
+		RootPath:             r.config.RootPath,
+		RunAs:                r.config.RunAs,
+		Schema:               r.config.Schema,
+		Serverless:           r.config.Serverless,
+		Storage:              r.config.Storage,
+		Tags:                 r.config.Tags,
+		Target:               r.config.Target,
+		Trigger:              r.config.Trigger,
+		PipelineId:           id,
+		ForceSendFields:      filterFields[pipelines.EditPipeline](r.config.ForceSendFields),
 	}
-	request.PipelineId = id
 
-	err = r.client.Pipelines.Update(ctx, request)
+	err := r.client.Pipelines.Update(ctx, request)
 	if err != nil {
 		return SDKError{Method: "Pipelines.Update", Err: err}
 	}

bundle/terranova/tnresources/schema.go

@@ -34,14 +34,16 @@ func (r *ResourceSchema) DoCreate(ctx context.Context) (string, error) {
 }
 
 func (r *ResourceSchema) DoUpdate(ctx context.Context, id string) error {
-	updateRequest := catalog.UpdateSchema{}
-	err := copyViaJSON(&updateRequest, r.config)
-	if err != nil {
-		return err
+	updateRequest := catalog.UpdateSchema{
+		Comment:                      r.config.Comment,
+		EnablePredictiveOptimization: "", // Not supported by DABs
+		FullName:                     id,
+		NewName:                      "", // We recreate schemas on name change intentionally.
+		Owner:                        "", // Not supported by DABs
+		Properties:                   r.config.Properties,
+		ForceSendFields:              filterFields[catalog.UpdateSchema](r.config.ForceSendFields),
 	}
 
-	updateRequest.FullName = id
-
 	response, err := r.client.Schemas.Update(ctx, updateRequest)
 	if err != nil {
 		return SDKError{Method: "Schemas.Update", Err: err}
@@ -55,7 +57,11 @@ func (r *ResourceSchema) DoUpdate(ctx context.Context, id string) error {
 }
 
 func DeleteSchema(ctx context.Context, client *databricks.WorkspaceClient, id string) error {
-	err := client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{FullName: id, Force: true})
+	err := client.Schemas.Delete(ctx, catalog.DeleteSchemaRequest{
+		FullName:        id,
+		Force:           true,
+		ForceSendFields: nil,
+	})
 	if err != nil {
 		return SDKError{Method: "Schemas.Delete", Err: err}
 	}

bundle/terranova/tnresources/sql_warehouse.go

@@ -35,12 +35,23 @@ func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context) (string, error) {
 }
 
 func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string) error {
-	request := sql.EditWarehouseRequest{}
-	err := copyViaJSON(&request, r.config)
-	if err != nil {
-		return err
+	request := sql.EditWarehouseRequest{
+		AutoStopMins:            r.config.AutoStopMins,
+		Channel:                 r.config.Channel,
+		ClusterSize:             r.config.ClusterSize,
+		CreatorName:             r.config.CreatorName,
+		EnablePhoton:            r.config.EnablePhoton,
+		EnableServerlessCompute: r.config.EnableServerlessCompute,
+		Id:                      id,
+		InstanceProfileArn:      r.config.InstanceProfileArn,
+		MaxNumClusters:          r.config.MaxNumClusters,
+		MinNumClusters:          r.config.MinNumClusters,
+		Name:                    r.config.Name,
+		SpotInstancePolicy:      r.config.SpotInstancePolicy,
+		Tags:                    r.config.Tags,
+		WarehouseType:           sql.EditWarehouseRequestWarehouseType(r.config.WarehouseType),
+		ForceSendFields:         filterFields[sql.EditWarehouseRequest](r.config.ForceSendFields),
 	}
-	request.Id = id
 
 	waiter, err := r.client.Warehouses.Edit(ctx, request)
 	if err != nil {

bundle/terranova/tnresources/util.go

@@ -1,24 +1,20 @@
 package tnresources
 
 import (
-	"encoding/json"
-	"errors"
-	"fmt"
+	"reflect"
 )
 
-// Utility to copy from one type to another based on intermediate JSON transformation
-// (e.g. to copy from JobSettings to CreateJob)
-func copyViaJSON[T1, T2 any](dest *T1, src T2) error {
-	if dest == nil {
-		return errors.New("internal error: unexpected nil")
-	}
-	data, err := json.Marshal(src)
-	if err != nil {
-		return fmt.Errorf("Failed to serialize %T: %w", src, err)
-	}
-	err = json.Unmarshal(data, dest)
-	if err != nil {
-		return fmt.Errorf("Failed JSON roundtrip from %T to %T: %w", src, dest, err)
+// filterFields creates a new slice with fields present only in the provided type.
+// We must use that when copying structs because JSON marshaller in SDK crashes if it sees unknown field.
+func filterFields[T any](fields []string) []string {
+	var result []string
+	typeOfT := reflect.TypeOf((*T)(nil)).Elem()
+
+	for _, field := range fields {
+		if _, ok := typeOfT.FieldByName(field); ok {
+			result = append(result, field)
+		}
 	}
-	return nil
+
+	return result
 }

bundle/terranova/tnresources/volume.go

@@ -41,6 +41,10 @@ func (r *ResourceVolume) DoUpdate(ctx context.Context, id string) error {
 	updateRequest := catalog.UpdateVolumeRequestContent{
 		Comment: r.config.Comment,
 		Name:    id,
+		NewName: "", // Not supported by Update(). Needs DoUpdateWithID()
+		Owner:   "", // Not supported by DABs
+
+		ForceSendFields: nil,
 	}
 
 	nameFromID, err := getNameFromID(id)
@@ -68,6 +72,11 @@ func (r *ResourceVolume) DoUpdateWithID(ctx context.Context, id string) (string,
 	updateRequest := catalog.UpdateVolumeRequestContent{
 		Comment: r.config.Comment,
 		Name:    id,
+
+		NewName: "", // Initialized below if needed
+		Owner:   "", // Not supported by DABs
+
+		ForceSendFields: nil,
 	}
 
 	items := strings.Split(id, ".")