fix(tests): Find empty columns for JSON types (#1418)

I'm not sure if there's a better way, but this seems to work for me.

~Still testing cases.~

Fixes cloudquery/cloudquery#15707

Author: disq

schema/validators.go

@@ -1,7 +1,10 @@
 package schema
 
 import (
+	"encoding/json"
+
 	"github.com/apache/arrow/go/v14/arrow"
+	"github.com/cloudquery/plugin-sdk/v4/types"
 )
 
 func FindEmptyColumns(table *Table, records []arrow.Record) []string {
@@ -12,6 +15,14 @@ func FindEmptyColumns(table *Table, records []arrow.Record) []string {
 		for colIndex, arr := range resource.Columns() {
 			for i := 0; i < arr.Len(); i++ {
 				if arr.IsValid(i) {
+					if arrow.TypeEqual(arr.DataType(), types.ExtensionTypes.JSON) {
+						// JSON column shouldn't be empty
+						val := arr.GetOneForMarshal(i).(json.RawMessage)
+						if isEmptyJSON(val) {
+							continue
+						}
+					}
+
 					columnsWithValues[colIndex] = true
 				}
 			}
@@ -28,3 +39,15 @@ func FindEmptyColumns(table *Table, records []arrow.Record) []string {
 	}
 	return emptyColumns
 }
+
+func isEmptyJSON(msg json.RawMessage) bool {
+	if len(msg) == 0 {
+		return true
+	}
+	switch string(msg) {
+	case "null", "{}", "[]":
+		return true
+	default:
+		return false
+	}
+}

schema/validators_test.go

@@ -0,0 +1,56 @@
+package schema
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/apache/arrow/go/v14/arrow"
+	"github.com/apache/arrow/go/v14/arrow/array"
+	"github.com/apache/arrow/go/v14/arrow/memory"
+	"github.com/cloudquery/plugin-sdk/v4/types"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFindEmptyColumns(t *testing.T) {
+	table := TestTable("test", TestSourceOptions{})
+	tg := NewTestDataGenerator()
+	record := tg.Generate(table, GenTestDataOptions{
+		MaxRows:  1,
+		NullRows: true,
+	})
+	v := FindEmptyColumns(table, []arrow.Record{record})
+	require.NotEmpty(t, v)
+	require.Len(t, v, len(table.Columns)-1) // exclude "id"
+}
+
+func TestFindEmptyColumnsNotEmpty(t *testing.T) {
+	table := TestTable("test", TestSourceOptions{})
+	tg := NewTestDataGenerator()
+	record := tg.Generate(table, GenTestDataOptions{
+		MaxRows:  1,
+		NullRows: false,
+	})
+	v := FindEmptyColumns(table, []arrow.Record{record})
+	require.Empty(t, v)
+}
+
+func TestFindEmptyColumnsJSON(t *testing.T) {
+	table := &Table{
+		Name: "test",
+		Columns: ColumnList{
+			{Name: "json", Type: types.ExtensionTypes.JSON},
+		},
+	}
+	sc := table.ToArrowSchema()
+	bldr := array.NewRecordBuilder(memory.DefaultAllocator, sc)
+	err := bldr.Field(0).UnmarshalJSON([]byte(`[{}]`))
+	if err != nil {
+		panic(fmt.Sprintf("failed to unmarshal json for column: %v", err))
+	}
+	records := []arrow.Record{bldr.NewRecord()}
+	bldr.Release()
+
+	v := FindEmptyColumns(table, records)
+	require.NotEmpty(t, v)
+	require.Len(t, v, 1)
+}