Skip to content

Commit 17285b9

Browse files
authored
fix: Validate all rows in a record (#2341)
#### Summary Found this while working on another internal issue cloudquery/cloudquery-private#9520 (comment). It's enough for a single row in a record to have a valid value to pass the validation. I think we should be strict and check all rows ---
1 parent 5d84688 commit 17285b9

File tree

1 file changed

+21
-10
lines changed

1 file changed

+21
-10
lines changed

schema/validators.go

Lines changed: 21 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,25 +9,36 @@ import (
99
"github.com/cloudquery/plugin-sdk/v4/types"
1010
)
1111

12+
func isValueValid(i int, arr arrow.Array) bool {
13+
if arr.IsValid(i) {
14+
if arrow.TypeEqual(arr.DataType(), types.ExtensionTypes.JSON) {
15+
// JSON column shouldn't be empty
16+
val := arr.GetOneForMarshal(i).(json.RawMessage)
17+
if isEmptyJSON(val) {
18+
return false
19+
}
20+
}
21+
return true
22+
}
23+
return false
24+
}
25+
1226
func FindEmptyColumns(table *Table, records []arrow.Record) []string {
1327
columnsWithValues := make([]bool, len(table.Columns))
1428
emptyColumns := make([]string, 0)
1529

1630
for _, resource := range records {
1731
for colIndex, arr := range resource.Columns() {
32+
allValuesValid := true
1833
for i := 0; i < arr.Len(); i++ {
19-
if arr.IsValid(i) {
20-
if arrow.TypeEqual(arr.DataType(), types.ExtensionTypes.JSON) {
21-
// JSON column shouldn't be empty
22-
val := arr.GetOneForMarshal(i).(json.RawMessage)
23-
if isEmptyJSON(val) {
24-
continue
25-
}
26-
}
27-
28-
columnsWithValues[colIndex] = true
34+
if !isValueValid(i, arr) {
35+
allValuesValid = false
36+
break
2937
}
3038
}
39+
if allValuesValid {
40+
columnsWithValues[colIndex] = true
41+
}
3142
}
3243
}
3344

0 commit comments

Comments
 (0)