add sensitive column validation

Author: blesniewski

plugin/testing_validation.go

@@ -14,5 +14,12 @@ func ValidateNoEmptyColumns(t *testing.T, tables schema.Tables, messages message
 		if len(emptyColumns) > 0 {
 			t.Fatalf("found empty column(s): %v in %s", emptyColumns, table.Name)
 		}
+		nonMatchingColumns, nonMatchingJsonColumns := schema.FindNotMatchingSensitiveColumns(table, records)
+		if len(nonMatchingColumns) > 0 {
+			t.Fatalf("found non-matching sensitive column(s): %v in %s", nonMatchingColumns, table.Name)
+		}
+		if len(nonMatchingJsonColumns) > 0 {
+			t.Fatalf("found non-matching sensitive JSON column(s): %v in %s", nonMatchingJsonColumns, table.Name)
+		}
 	}
 }

schema/validators.go

@@ -2,6 +2,8 @@ package schema
 
 import (
 	"encoding/json"
+	"slices"
+	"strings"
 
 	"github.com/apache/arrow-go/v18/arrow"
 	"github.com/cloudquery/plugin-sdk/v4/types"
@@ -40,6 +42,36 @@ func FindEmptyColumns(table *Table, records []arrow.Record) []string {
 	return emptyColumns
 }
 
+func FindNotMatchingSensitiveColumns(table *Table, records []arrow.Record) ([]string, []string) {
+	if len(table.SensitiveColumns) == 0 {
+		return []string{}, []string{}
+	}
+
+	nonMatchingColumns := make([]string, 0)
+	nonMatchingJsonColumns := make([]string, 0)
+	tableColumns := table.Columns.Names()
+	for _, c := range table.SensitiveColumns {
+		isJsonPath := false
+		if strings.Contains(c, ".") {
+			c = strings.Split(c, ".")[0]
+			isJsonPath = true
+		}
+		if !slices.Contains(tableColumns, c) {
+			nonMatchingColumns = append(nonMatchingColumns, c)
+			continue
+		}
+		if !isJsonPath {
+			continue
+		}
+		col := table.Columns.Get(c)
+		if !arrow.TypeEqual(col.Type, types.ExtensionTypes.JSON) {
+			nonMatchingJsonColumns = append(nonMatchingJsonColumns, c)
+			continue
+		}
+	}
+	return nonMatchingColumns, nonMatchingJsonColumns
+}
+
 func isEmptyJSON(msg json.RawMessage) bool {
 	if len(msg) == 0 {
 		return true