feat(codegen): Add WithTypeTransformer (#157)

This is take two of #139

Trying to get right the `WithTypeTransformer` so we wont have to change
it in the near future as it's heavilly used external API.

I've also added error handling to all functions as we were basically
ignoring all of them and printing to the screen is not the right
approach catching those locally and in CI.

---

Use the following steps to ensure your PR is ready to be reviewed

- [ ] Read the [contribution guidelines](../blob/main/CONTRIBUTING.md)
🧑‍🎓
- [ ] Run `go fmt` to format your code 🖊
- [ ] Lint your changes via `golangci-lint run` 🚨 (install golangci-lint
[here](https://golangci-lint.run/usage/install/#local-installation))
- [ ] Update or add tests 🧪
- [ ] Ensure the status checks below are successful ✅

Co-authored-by: erezrokah <[email protected]>
Co-authored-by: Herman Schaaf <[email protected]>

Author: 3 people

codegen/golang.go

@@ -11,6 +11,7 @@ import (
 	"unicode"
 
 	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2"
 	"github.com/iancoleman/strcase"
 )
 
@@ -19,9 +20,11 @@ type TableOptions func(*TableDefinition)
 //go:embed templates/*.go.tpl
 var TemplatesFS embed.FS
 
-func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
+func DefaultGoTypeToSchemaType(v reflect.Type) (schema.ValueType, error) {
 	k := v.Kind()
 	switch k {
+	case reflect.Pointer:
+		return DefaultGoTypeToSchemaType(v.Elem())
 	case reflect.String:
 		return schema.TypeString, nil
 	case reflect.Bool:
@@ -39,8 +42,6 @@ func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
 			return schema.TypeTimestamp, nil
 		}
 		return schema.TypeJSON, nil
-	case reflect.Pointer:
-		return valueToSchemaType(v.Elem())
 	case reflect.Slice:
 		switch v.Elem().Kind() {
 		case reflect.String:
@@ -56,7 +57,11 @@ func valueToSchemaType(v reflect.Type) (schema.ValueType, error) {
 	}
 }
 
-func WithNameTransformer(transformer func(field reflect.StructField) string) TableOptions {
+func DefaultTypeTransformer(v reflect.StructField) (schema.ValueType, error) {
+	return DefaultGoTypeToSchemaType(v.Type)
+}
+
+func WithNameTransformer(transformer NameTransformer) TableOptions {
 	return func(t *TableDefinition) {
 		t.nameTransformer = transformer
 	}
@@ -81,23 +86,37 @@ func WithUnwrapFieldsStructs(fields []string) TableOptions {
 	}
 }
 
-// Unwrap all fields that are embedded structs (1 level deep only)
+// WithUnwrapAllEmbeddedStructs unwraps all fields that are embedded structs (1 level deep only)
 func WithUnwrapAllEmbeddedStructs() TableOptions {
 	return func(t *TableDefinition) {
 		t.unwrapAllEmbeddedStructFields = true
 	}
 }
 
-func DefaultTransformer(field reflect.StructField) string {
+// WithLogger
+func WithLogger(logger zerolog.Logger) TableOptions {
+	return func(t *TableDefinition) {
+		t.logger = logger
+	}
+}
+
+// WithValueTypeTransformer sets a function that can override the schema type for specific fields. Return `schema.TypeInvalid` to fall back to default behavior.
+func WithTypeTransformer(transformer TypeTransformer) TableOptions {
+	return func(t *TableDefinition) {
+		t.typeTransformer = transformer
+	}
+}
+
+func DefaultTransformer(field reflect.StructField) (string, error) {
 	name := field.Name
 	if jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]; len(jsonTag) > 0 {
 		// return empty string if the field is not related api response
 		if jsonTag == "-" {
-			return ""
+			return "", nil
 		}
 		name = jsonTag
 	}
-	return strcase.ToSnake(name)
+	return strcase.ToSnake(name), nil
 }
 
 func sliceContains(arr []string, s string) bool {
@@ -139,27 +158,40 @@ func (t *TableDefinition) ignoreField(field reflect.StructField) bool {
 	return len(field.Name) == 0 || unicode.IsLower(rune(field.Name[0])) || sliceContains(t.skipFields, field.Name)
 }
 
-func (t *TableDefinition) addColumnFromField(field reflect.StructField, parent *reflect.StructField) {
+func (t *TableDefinition) addColumnFromField(field reflect.StructField, parent *reflect.StructField) error {
 	if t.ignoreField(field) {
-		return
+		return nil
 	}
 
-	columnType, err := valueToSchemaType(field.Type)
+	columnType, err := t.typeTransformer(field)
 	if err != nil {
-		fmt.Printf("skipping field %s on table %s, got err: %v\n", field.Name, t.Name, err)
-		return
+		return fmt.Errorf("failed to transform type for field %s: %w", field.Name, err)
+	}
+
+	if columnType == schema.TypeInvalid {
+		columnType, err = DefaultTypeTransformer(field)
+		if err != nil {
+			return fmt.Errorf("failed to transform type for field %s: %w", field.Name, err)
+		}
 	}
 
 	// generate a PathResolver to use by default
 	pathResolver := fmt.Sprintf(`schema.PathResolver("%s")`, field.Name)
-	name := t.nameTransformer(field)
+	name, err := t.nameTransformer(field)
+	if err != nil {
+		return fmt.Errorf("failed to transform field name for field %s: %w", field.Name, err)
+	}
 	// skip field if there is no name
 	if name == "" {
-		return
+		return nil
 	}
 	if parent != nil {
 		pathResolver = fmt.Sprintf(`schema.PathResolver("%s.%s")`, parent.Name, field.Name)
-		name = t.nameTransformer(*parent) + "_" + name
+		parentName, err := t.nameTransformer(*parent)
+		if err != nil {
+			return fmt.Errorf("failed to transform field name for parent field %s: %w", parent.Name, err)
+		}
+		name = fmt.Sprintf("%s_%s", parentName, name)
 	}
 
 	column := ColumnDefinition{
@@ -168,13 +200,15 @@ func (t *TableDefinition) addColumnFromField(field reflect.StructField, parent *
 		Resolver: pathResolver,
 	}
 	t.Columns = append(t.Columns, column)
+	return nil
 }
 
 // NewTableFromStruct creates a new TableDefinition from a struct by inspecting its fields
 func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*TableDefinition, error) {
 	t := &TableDefinition{
 		Name:            name,
 		nameTransformer: DefaultTransformer,
+		typeTransformer: DefaultTypeTransformer,
 	}
 	for _, opt := range opts {
 		opt(t)
@@ -201,10 +235,14 @@ func NewTableFromStruct(name string, obj interface{}, opts ...TableOptions) (*Ta
 				parent = &field
 			}
 			for _, f := range unwrappedFields {
-				t.addColumnFromField(f, parent)
+				if err := t.addColumnFromField(f, parent); err != nil {
+					return nil, fmt.Errorf("failed to add column from field %s: %w", f.Name, err)
+				}
 			}
 		} else {
-			t.addColumnFromField(field, nil)
+			if err := t.addColumnFromField(field, nil); err != nil {
+				return nil, fmt.Errorf("failed to add column for field %s: %w", field.Name, err)
+			}
 		}
 	}
 

codegen/golang_test.go

@@ -3,6 +3,7 @@ package codegen
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
 
@@ -48,6 +49,10 @@ type testStructWithNonEmbeddedStruct struct {
 	NonEmbedded *embeddedStruct
 }
 
+type testStructWithCustomType struct {
+	TimeCol time.Time `json:"time_col,omitempty"`
+}
+
 var expectedColumns = []ColumnDefinition{
 	{
 		Name:     "int_col",
@@ -156,13 +161,31 @@ func TestTableFromGoStruct(t *testing.T) {
 			want: expectedTestTableEmbeddedStruct,
 		},
 		{
-			name: "should_unwrap_specific_structs_when_option_is_set",
+			name: "should unwrap specific structs when option is set",
 			args: args{
 				testStruct: testStructWithNonEmbeddedStruct{},
 				options:    []TableOptions{WithUnwrapFieldsStructs([]string{"NonEmbedded"})},
 			},
 			want: expectedTestTableNonEmbeddedStruct,
 		},
+		{
+			name: "should override schema type when option is set",
+			args: args{
+				testStruct: testStructWithCustomType{},
+				options: []TableOptions{WithTypeTransformer(func(t reflect.StructField) (schema.ValueType, error) {
+					switch t.Type {
+					case reflect.TypeOf(time.Time{}), reflect.TypeOf(&time.Time{}):
+						return schema.TypeJSON, nil
+					default:
+						return schema.TypeInvalid, nil
+					}
+				})},
+			},
+			want: TableDefinition{Name: "test_struct",
+				// We expect the time column to be of type JSON, since we override the type of `time.Time` to be JSON
+				Columns:         ColumnDefinitions{{Name: "time_col", Type: schema.TypeJSON, Resolver: `schema.PathResolver("TimeCol")`}},
+				nameTransformer: DefaultTransformer},
+		},
 	}
 
 	for _, tt := range tests {

codegen/table.go

@@ -4,13 +4,17 @@ import (
 	"reflect"
 
 	"github.com/cloudquery/plugin-sdk/schema"
+	"github.com/grpc-ecosystem/go-grpc-middleware/providers/zerolog/v2"
 )
 
 type ResourceDefinition struct {
 	Name  string
 	Table *TableDefinition
 }
 
+type NameTransformer func(reflect.StructField) (string, error)
+type TypeTransformer func(reflect.StructField) (schema.ValueType, error)
+
 type TableDefinition struct {
 	Name        string
 	Columns     ColumnDefinitions
@@ -22,11 +26,13 @@ type TableDefinition struct {
 	Multiplex                     string
 	PostResourceResolver          string
 	PreResourceResolver           string
-	nameTransformer               func(reflect.StructField) string
+	nameTransformer               NameTransformer
+	typeTransformer               TypeTransformer
 	skipFields                    []string
 	extraColumns                  ColumnDefinitions
 	structFieldsToUnwrap          []string
 	unwrapAllEmbeddedStructFields bool
+	logger                        zerolog.Logger
 }
 
 type ColumnDefinitions []ColumnDefinition