fix: Conversion and test fixes (#1064)

Some fixes to the scalar and testing code:
- use `AppendTime` in `AppendToBuilder` to take different time units
into account
- base64 decode binary-encoded JSON strings when the type calls for it
in struct `Set()`, otherwise the binary data gets interpreted as a
string and the results are incorrect
- exclude the JSON type from list tests, we didn't support that before
and don't need to introduce it now

Author: hermanschaaf

scalar/scalar.go

@@ -182,7 +182,7 @@ func AppendToBuilder(bldr array.Builder, s Scalar) {
 	case arrow.BOOL:
 		bldr.(*array.BooleanBuilder).Append(s.(*Bool).Value)
 	case arrow.TIMESTAMP:
-		bldr.(*array.TimestampBuilder).Append(arrow.Timestamp(s.(*Timestamp).Value.UnixMicro()))
+		bldr.(*array.TimestampBuilder).AppendTime(s.(*Timestamp).Value)
 	case arrow.DURATION:
 		bldr.(*array.DurationBuilder).Append(arrow.Duration(s.(*Duration).Value))
 	case arrow.DATE32:
@@ -217,7 +217,6 @@ func AppendToBuilder(bldr array.Builder, s Scalar) {
 		st := sb.Type().(*arrow.StructType)
 		for i, f := range st.Fields() {
 			sc := NewScalar(sb.FieldBuilder(i).Type())
-
 			if sv, ok := m[f.Name]; ok {
 				if err := sc.Set(sv); err != nil {
 					panic(err)

scalar/struct.go

@@ -1,6 +1,7 @@
 package scalar
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"reflect"
 
@@ -61,6 +62,28 @@ func (s *Struct) Set(val any) error {
 		if err := json.Unmarshal([]byte(value), &x); err != nil {
 			return err
 		}
+		for name := range x {
+			if f, ok := s.Type.FieldByName(name); ok {
+				xs, ok := x[name].(string)
+				if !ok {
+					continue
+				}
+				switch {
+				case arrow.TypeEqual(f.Type, arrow.BinaryTypes.Binary):
+					v, err := base64.StdEncoding.DecodeString(xs)
+					if err != nil {
+						return err
+					}
+					x[name] = v
+				case arrow.TypeEqual(f.Type, arrow.BinaryTypes.LargeBinary):
+					v, err := base64.StdEncoding.DecodeString(xs)
+					if err != nil {
+						return err
+					}
+					x[name] = v
+				}
+			}
+		}
 		s.Value = x
 
 	case []byte:

scalar/struct_test.go

@@ -1,15 +1,55 @@
 package scalar
 
 import (
+	"encoding/json"
 	"strconv"
 	"testing"
 
 	"github.com/apache/arrow/go/v13/arrow"
 	"github.com/apache/arrow/go/v13/arrow/array"
 	"github.com/apache/arrow/go/v13/arrow/memory"
+	"github.com/cloudquery/plugin-sdk/v4/types"
 	"github.com/stretchr/testify/assert"
 )
 
+func TestStructEncodeDecode(t *testing.T) {
+	tl := []struct {
+		name  string
+		dt    *arrow.StructType
+		input any
+	}{
+		{name: "binary", dt: arrow.StructOf(arrow.Field{Name: "binary", Type: arrow.BinaryTypes.Binary}), input: `{"binary":"7049Ug=="}`},
+		{name: "uuid", dt: arrow.StructOf(arrow.Field{Name: "uuid", Type: types.ExtensionTypes.UUID}), input: `{"uuid":"f81d4fae-7dec-11d0-a765-00a0c91e6bf6"}`},
+	}
+
+	for _, tc := range tl {
+		t.Run(tc.name, func(t *testing.T) {
+			bldr := array.NewBuilder(memory.DefaultAllocator, tc.dt)
+			defer bldr.Release()
+
+			s := NewScalar(tc.dt)
+			if !arrow.TypeEqual(s.DataType(), tc.dt) {
+				t.Fatalf("expected %v, got %v", tc.dt, s.DataType())
+			}
+
+			assert.NoError(t, s.Set(tc.input))
+			if t.Failed() {
+				return
+			}
+
+			assert.True(t, s.IsValid())
+			AppendToBuilder(bldr, s)
+			arr := bldr.NewArray()
+			one := arr.GetOneForMarshal(0)
+			json, err := json.Marshal(one)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.Equal(t, tc.input, string(json))
+		})
+	}
+}
+
 func TestStructMissingKeys(t *testing.T) {
 	tl := []struct {
 		dt          *arrow.StructType

schema/testdata.go

@@ -89,10 +89,10 @@ func TestTable(name string, opts TestSourceOptions) *Table {
 		{Name: "boolean", Type: arrow.FixedWidthTypes.Boolean},
 
 		// extension types
-		{Name: "uuid", Type: types.NewUUIDType()},
-		{Name: "inet", Type: types.NewInetType()},
-		{Name: "mac", Type: types.NewMACType()},
-		{Name: "json", Type: types.NewJSONType()},
+		{Name: "uuid", Type: types.ExtensionTypes.UUID},
+		{Name: "inet", Type: types.ExtensionTypes.Inet},
+		{Name: "mac", Type: types.ExtensionTypes.MAC},
+		{Name: "json", Type: types.ExtensionTypes.JSON},
 	}...)
 	if !opts.SkipDates {
 		columns = append(columns, ColumnList{
@@ -157,7 +157,8 @@ func TestTable(name string, opts TestSourceOptions) *Table {
 	}
 
 	if !opts.SkipLists {
-		columns = append(columns, listOfColumns(columns)...)
+		cols := excludeType(columns, types.ExtensionTypes.JSON)
+		columns = append(columns, listOfColumns(cols)...)
 	}
 
 	if !opts.SkipMaps {
@@ -168,6 +169,16 @@ func TestTable(name string, opts TestSourceOptions) *Table {
 	return t
 }
 
+func excludeType(columns ColumnList, typ arrow.DataType) ColumnList {
+	var cols ColumnList
+	for _, c := range columns {
+		if c.Type.ID() != typ.ID() {
+			cols = append(cols, c)
+		}
+	}
+	return cols
+}
+
 // var PKColumnNames = []string{"uuid_pk"}
 
 // GenTestDataOptions are options for generating test data