Migrate SQL Warehouse to Go SDK (#3044)

* Migrate SQL Warehouse to Go SDK

* fix compile

* check in diff-schema scrip

* fix

* fix

* test fix

* fix

* fix

* undo extra change

* work

* work

* work

* fix

* fixes

* fix

* remove

* handle slices as well

* fix stubs in exporter test

* test

* Fix tags diff suppression

* fix

* Address comments

* comments

* fix workflow

* fix

* only diff against merge-base of master

* update docs

* remove deprecated resources

* undo extra docs changes

* remove extra space

* small tweak

* small tweaks

* tests and address comments

Author: mgyucht

common/reflect_resource.go

@@ -77,13 +77,44 @@ func MustSchemaPath(s map[string]*schema.Schema, path ...string) *schema.Schema
 // StructToSchema makes schema from a struct type & applies customizations from callback given
 func StructToSchema(v any, customize func(map[string]*schema.Schema) map[string]*schema.Schema) map[string]*schema.Schema {
 	rv := reflect.ValueOf(v)
-	scm := typeToSchema(rv, rv.Type(), []string{})
+	scm := typeToSchema(rv, []string{})
 	if customize != nil {
 		scm = customize(scm)
 	}
 	return scm
 }
 
+// SetSuppressDiff adds diff suppression to a schema. This is necessary for non-computed
+// fields for which the platform returns a value, but the user has not configured any value.
+// For example: the REST API returns `{"tags": {}}` for a resource with no tags.
+func SetSuppressDiff(v *schema.Schema) {
+	v.DiffSuppressFunc = diffSuppressor(fmt.Sprintf("%v", v.Type.Zero()))
+}
+
+// SetDefault sets the default value for a schema.
+func SetDefault(v *schema.Schema, value any) {
+	v.Default = value
+	v.Optional = true
+	v.Required = false
+}
+
+// SetReadOnly sets the schema to be read-only (i.e. computed, non-optional).
+// This should be used for fields that are not user-configurable but are returned
+// by the platform.
+func SetReadOnly(v *schema.Schema) {
+	v.Optional = false
+	v.Required = false
+	v.MaxItems = 0
+	v.Computed = true
+}
+
+// SetRequired sets the schema to be required.
+func SetRequired(v *schema.Schema) {
+	v.Optional = false
+	v.Required = true
+	v.Computed = false
+}
+
 func isOptional(typeField reflect.StructField) bool {
 	if strings.Contains(typeField.Tag.Get("json"), "omitempty") {
 		return true
@@ -175,26 +206,52 @@ func diffSuppressor(zero string) func(k, old, new string, d *schema.ResourceData
 			log.Printf("[DEBUG] Suppressing diff for %v: platform=%#v config=%#v", k, old, new)
 			return true
 		}
+		if strings.HasSuffix(k, ".#") && new == "0" && old != "0" {
+			field := strings.TrimSuffix(k, ".#")
+			log.Printf("[DEBUG] Suppressing diff for list or set %v: no value configured but platform returned some value (likely {})", field)
+			return true
+		}
 		return false
 	}
 }
 
+type field struct {
+	sf reflect.StructField
+	v  reflect.Value
+}
+
+func listAllFields(v reflect.Value) []field {
+	t := v.Type()
+	fields := make([]field, 0, v.NumField())
+	for i := 0; i < v.NumField(); i++ {
+		f := t.Field(i)
+		if f.Anonymous {
+			fields = append(fields, listAllFields(v.Field(i))...)
+		} else {
+			fields = append(fields, field{
+				sf: f,
+				v:  v.Field(i),
+			})
+		}
+	}
+	return fields
+}
+
 // typeToSchema converts struct into terraform schema. `path` is used for block suppressions
 // special path element `"0"` is used to denote either arrays or sets of elements
-func typeToSchema(v reflect.Value, t reflect.Type, path []string) map[string]*schema.Schema {
+func typeToSchema(v reflect.Value, path []string) map[string]*schema.Schema {
 	scm := map[string]*schema.Schema{}
 	rk := v.Kind()
 	if rk == reflect.Ptr {
 		v = v.Elem()
-		t = v.Type()
 		rk = v.Kind()
 	}
 	if rk != reflect.Struct {
 		panic(fmt.Errorf("Schema value of Struct is expected, but got %s: %#v", reflectKind(rk), v))
 	}
-	for i := 0; i < v.NumField(); i++ {
-		typeField := t.Field(i)
-
+	fields := listAllFields(v)
+	for _, field := range fields {
+		typeField := field.sf
 		tfTag := typeField.Tag.Get("tf")
 
 		fieldName := chooseFieldName(typeField)
@@ -260,7 +317,7 @@ func typeToSchema(v reflect.Value, t reflect.Type, path []string) map[string]*sc
 			scm[fieldName].Type = schema.TypeList
 			elem := typeField.Type.Elem()
 			sv := reflect.New(elem).Elem()
-			nestedSchema := typeToSchema(sv, elem, append(path, fieldName, "0"))
+			nestedSchema := typeToSchema(sv, append(path, fieldName, "0"))
 			if strings.Contains(tfTag, "suppress_diff") {
 				blockCount := strings.Join(append(path, fieldName, "#"), ".")
 				scm[fieldName].DiffSuppressFunc = makeEmptyBlockSuppressFunc(blockCount)
@@ -279,7 +336,7 @@ func typeToSchema(v reflect.Value, t reflect.Type, path []string) map[string]*sc
 			elem := typeField.Type  // changed from ptr
 			sv := reflect.New(elem) // changed from ptr
 
-			nestedSchema := typeToSchema(sv, elem, append(path, fieldName, "0"))
+			nestedSchema := typeToSchema(sv, append(path, fieldName, "0"))
 			if strings.Contains(tfTag, "suppress_diff") {
 				blockCount := strings.Join(append(path, fieldName, "#"), ".")
 				scm[fieldName].DiffSuppressFunc = makeEmptyBlockSuppressFunc(blockCount)
@@ -310,7 +367,7 @@ func typeToSchema(v reflect.Value, t reflect.Type, path []string) map[string]*sc
 			case reflect.Struct:
 				sv := reflect.New(elem).Elem()
 				scm[fieldName].Elem = &schema.Resource{
-					Schema: typeToSchema(sv, elem, append(path, fieldName, "0")),
+					Schema: typeToSchema(sv, append(path, fieldName, "0")),
 				}
 			}
 		default:
@@ -341,6 +398,20 @@ func IsRequestEmpty(v any) (bool, error) {
 	return !isNotEmpty, err
 }
 
+// isGoSdk returns true if the struct is from databricks-sdk-go or embeds a struct from databricks-sdk-go.
+func isGoSdk(v reflect.Value) bool {
+	if strings.Contains(v.Type().PkgPath(), "databricks-sdk-go") {
+		return true
+	}
+	for i := 0; i < v.NumField(); i++ {
+		f := v.Type().Field(i)
+		if f.Anonymous && isGoSdk(v.Field(i)) {
+			return true
+		}
+	}
+	return false
+}
+
 func iterFields(rv reflect.Value, path []string, s map[string]*schema.Schema,
 	cb func(fieldSchema *schema.Schema, path []string, valueField *reflect.Value) error) error {
 	rk := rv.Kind()
@@ -350,9 +421,10 @@ func iterFields(rv reflect.Value, path []string, s map[string]*schema.Schema,
 	if !rv.IsValid() {
 		return fmt.Errorf("%s: got invalid reflect value %#v", path, rv)
 	}
-	isGoSDK := strings.Contains(rv.Type().PkgPath(), "databricks-sdk-go")
-	for i := 0; i < rv.NumField(); i++ {
-		typeField := rv.Type().Field(i)
+	isGoSDK := isGoSdk(rv)
+	fields := listAllFields(rv)
+	for _, field := range fields {
+		typeField := field.sf
 		fieldName := chooseFieldName(typeField)
 		if fieldName == "-" {
 			continue
@@ -370,7 +442,7 @@ func iterFields(rv reflect.Value, path []string, s map[string]*schema.Schema,
 		if fieldSchema.Optional && defaultEmpty && !omitEmpty {
 			return fmt.Errorf("inconsistency: %s is optional, default is empty, but has no omitempty", fieldName)
 		}
-		valueField := rv.Field(i)
+		valueField := field.v
 		err := cb(fieldSchema, append(path, fieldName), &valueField)
 		if err != nil {
 			return fmt.Errorf("%s: %s", fieldName, err)
@@ -393,13 +465,15 @@ func collectionToMaps(v any, s *schema.Schema) ([]any, error) {
 		return nil, fmt.Errorf("not resource")
 	}
 	var allItems []reflect.Value
-	if s.MaxItems == 1 {
-		allItems = append(allItems, reflect.ValueOf(v))
-	} else {
-		vs := reflect.ValueOf(v)
-		for i := 0; i < vs.Len(); i++ {
-			allItems = append(allItems, vs.Index(i))
+	rv := reflect.ValueOf(v)
+	rvType := rv.Type().Kind()
+	isList := rvType == reflect.Array || rvType == reflect.Slice
+	if isList {
+		for i := 0; i < rv.Len(); i++ {
+			allItems = append(allItems, rv.Index(i))
 		}
+	} else {
+		allItems = append(allItems, rv)
 	}
 	for _, v := range allItems {
 		data := map[string]any{}

common/reflect_resource_test.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/databricks/databricks-sdk-go/service/sql"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/stretchr/testify/assert"
 )
@@ -444,7 +445,7 @@ func TestTypeToSchemaNoStruct(t *testing.T) {
 			fmt.Sprintf("%s", p))
 	}()
 	v := reflect.ValueOf(1)
-	typeToSchema(v, v.Type(), []string{})
+	typeToSchema(v, []string{})
 }
 
 func TestTypeToSchemaUnsupported(t *testing.T) {
@@ -457,7 +458,7 @@ func TestTypeToSchemaUnsupported(t *testing.T) {
 		New chan int `json:"new"`
 	}
 	v := reflect.ValueOf(nonsense{})
-	typeToSchema(v, v.Type(), []string{})
+	typeToSchema(v, []string{})
 }
 
 type data map[string]any
@@ -634,3 +635,79 @@ func TestDataResourceWithID(t *testing.T) {
 	assert.Equal(t, "out: id", d.Get("out"))
 	assert.Equal(t, "abc", d.Id())
 }
+
+func TestStructToSchema_go_sdk_embedded(t *testing.T) {
+	type T struct {
+		sql.GetWarehouseResponse
+		Extra string `json:"extra,omitempty" tf:"computed"`
+	}
+	s := StructToSchema(T{}, nil)
+	autoStopMins, ok := s["auto_stop_mins"]
+	assert.True(t, ok)
+	assert.Equal(t, schema.TypeInt, autoStopMins.Type)
+	assert.True(t, autoStopMins.Optional)
+}
+
+func TestStructToData_go_sdk_embedded(t *testing.T) {
+	type T struct {
+		sql.GetWarehouseResponse
+		Extra string `json:"extra,omitempty" tf:"computed"`
+	}
+	s := StructToSchema(T{}, nil)
+	SetRequired(s["cluster_size"])
+	r := &schema.Resource{
+		Schema: s,
+	}
+	d := r.TestResourceData()
+	d.Set("cluster_size", "original")
+	err := StructToData(T{
+		GetWarehouseResponse: sql.GetWarehouseResponse{
+			ClusterSize: "abc123",
+		},
+		Extra: "extra",
+	}, s, d)
+	assert.NoError(t, err)
+	assert.Equal(t, "abc123", d.Get("cluster_size"))
+	assert.Equal(t, "extra", d.Get("extra"))
+}
+
+func TestStructToSchema_go_sdk_field(t *testing.T) {
+	type T struct {
+		Warehouse *sql.GetWarehouseResponse `json:"warehouse,omitempty"`
+		Extra     string                    `json:"extra,omitempty" tf:"computed"`
+	}
+	s := StructToSchema(T{}, nil)
+	warehouse, ok := s["warehouse"]
+	assert.True(t, ok)
+	autoStopMins, ok := warehouse.Elem.(*schema.Resource).Schema["auto_stop_mins"]
+	assert.True(t, ok)
+	assert.Equal(t, schema.TypeInt, autoStopMins.Type)
+	assert.True(t, autoStopMins.Optional)
+}
+
+func TestStructToData_go_sdk_field(t *testing.T) {
+	type T struct {
+		Warehouse *sql.GetWarehouseResponse `json:"warehouse,omitempty"`
+		Extra     string                    `json:"extra,omitempty" tf:"computed"`
+	}
+	s := StructToSchema(T{}, nil)
+	SetRequired(MustSchemaPath(s, "warehouse", "cluster_size"))
+	r := &schema.Resource{
+		Schema: s,
+	}
+	d := r.TestResourceData()
+	d.Set("warehouse", []map[string]any{
+		{
+			"cluster_size": "original",
+		},
+	})
+	err := StructToData(T{
+		Warehouse: &sql.GetWarehouseResponse{
+			ClusterSize: "abc123",
+		},
+		Extra: "extra",
+	}, s, d)
+	assert.NoError(t, err)
+	assert.Equal(t, "abc123", d.Get("warehouse.0.cluster_size"))
+	assert.Equal(t, "extra", d.Get("extra"))
+}