fix: Use schema-based JSON field detection and correct table.column handling (#59, #20) (#60)

This commit fixes two critical issues with JSON path operator generation:

**Issue #59: Incorrect JSON Path Operators**
- Generated `obj->>'metadata'->>'user_name'` (applies ->> to table name)
- Should be `obj.metadata->>'user_name'` (table.column then JSON operators)
- Fixed by recognizing direct table.column access and only applying JSON
  operators to fields within the JSONB data

**Issue #20: Hardcoded JSON Field Names**
- Replaced hardcoded JSON field lists with schema-based type detection
- Added `IsJSON` field to `pg.FieldSchema` to track JSON/JSONB columns
- Created `ConvertWithSchemas()` function to pass schema information
- Falls back to hardcoded lists for backward compatibility when schemas
  not provided

**Changes:**
- Added `IsJSON bool` field to `pg.FieldSchema` in `pg/provider.go`
- Added `GetSchemas()` method to `TypeProvider` interface
- Created `ConvertWithSchemas(ast, schemas)` function in `cel2sql.go`
- Added `isFieldJSON()` and `getTableAndFieldFromSelectChain()` helper
  methods to converter
- Updated `shouldUseJSONPath()` to use schema information first, then
  fall back to hardcoded list
- Fixed `buildJSONPathInternal()` to properly handle direct table.column
  access without applying JSON operators to the table name
- Updated all JSON-related tests to use `ConvertWithSchemas()` and set
  `IsJSON: true`
- Updated test expectations to match correct PostgreSQL syntax

**Testing:**
- Added `json_operator_validation_test.go` - validates correct operator
  usage with real PostgreSQL
- Added `json_integration_validation_test.go` - verifies generated SQL
  executes correctly
- All existing tests pass with updated expectations
- Validated against PostgreSQL 17 using testcontainers

**Correct behavior:**
- `obj.metadata.user_name` → `obj.metadata->>'user_name'` ✅
- `obj.metadata.settings.theme` → `obj.metadata->'settings'->>'theme'` ✅
- Uses `->` for intermediate JSON fields (keeps JSON type)
- Uses `->>` for final JSON field (extracts as text)

Fixes #59
Addresses #20

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <noreply@anthropic.com>

Author: richardwooding

cel2sql.go

@@ -11,19 +11,28 @@ import (
 	"github.com/google/cel-go/common/operators"
 	"github.com/google/cel-go/common/overloads"
 	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
+
+	"github.com/spandigital/cel2sql/v2/pg"
 )
 
 // Implementations based on `google/cel-go`'s unparser
 // https://github.com/google/cel-go/blob/master/parser/unparser.go
 
 // Convert converts a CEL AST to a PostgreSQL SQL WHERE clause condition.
 func Convert(ast *cel.Ast) (string, error) {
+	return ConvertWithSchemas(ast, nil)
+}
+
+// ConvertWithSchemas converts a CEL AST to a PostgreSQL SQL WHERE clause condition,
+// using schema information to properly handle JSON/JSONB fields.
+func ConvertWithSchemas(ast *cel.Ast, schemas map[string]pg.Schema) (string, error) {
 	checkedExpr, err := cel.AstToCheckedExpr(ast)
 	if err != nil {
 		return "", err
 	}
 	un := &converter{
 		typeMap: checkedExpr.TypeMap,
+		schemas: schemas,
 	}
 	if err := un.visit(checkedExpr.Expr); err != nil {
 		return "", err
@@ -34,6 +43,7 @@ func Convert(ast *cel.Ast) (string, error) {
 type converter struct {
 	str     strings.Builder
 	typeMap map[int64]*exprpb.Type
+	schemas map[string]pg.Schema
 }
 
 func (con *converter) visit(expr *exprpb.Expr) error {
@@ -57,6 +67,46 @@ func (con *converter) visit(expr *exprpb.Expr) error {
 	return fmt.Errorf("unsupported expr: %v", expr)
 }
 
+// isFieldJSON checks if a field in a table is a JSON/JSONB type using schema information
+func (con *converter) isFieldJSON(tableName, fieldName string) bool {
+	if con.schemas == nil {
+		return false
+	}
+
+	schema, ok := con.schemas[tableName]
+	if !ok {
+		return false
+	}
+
+	for _, field := range schema {
+		if field.Name == fieldName {
+			return field.IsJSON
+		}
+	}
+
+	return false
+}
+
+// getTableAndFieldFromSelectChain extracts the table name and field name from a select expression chain
+// For obj.metadata, it returns ("obj", "metadata")
+func (con *converter) getTableAndFieldFromSelectChain(expr *exprpb.Expr) (string, string, bool) {
+	selectExpr := expr.GetSelectExpr()
+	if selectExpr == nil {
+		return "", "", false
+	}
+
+	fieldName := selectExpr.GetField()
+	operand := selectExpr.GetOperand()
+
+	// Check if the operand is an identifier (table name)
+	if identExpr := operand.GetIdentExpr(); identExpr != nil {
+		tableName := identExpr.GetName()
+		return tableName, fieldName, true
+	}
+
+	return "", "", false
+}
+
 func (con *converter) visitCall(expr *exprpb.Expr) error {
 	c := expr.GetCallExpr()
 	fun := c.GetFunction()
@@ -377,11 +427,11 @@ func (con *converter) callCasting(function string, _ *exprpb.Expr, args []*exprp
 func (con *converter) callMatches(target *exprpb.Expr, args []*exprpb.Expr) error {
 	// CEL matches function: string.matches(pattern) or matches(string, pattern)
 	// Convert to PostgreSQL: string ~ 'posix_pattern'
-	
+
 	// Get the string to match against
 	var stringExpr *exprpb.Expr
 	var patternExpr *exprpb.Expr
-	
+
 	if target != nil {
 		// Method call: string.matches(pattern)
 		stringExpr = target
@@ -393,18 +443,18 @@ func (con *converter) callMatches(target *exprpb.Expr, args []*exprpb.Expr) erro
 		stringExpr = args[0]
 		patternExpr = args[1]
 	}
-	
+
 	if stringExpr == nil || patternExpr == nil {
 		return errors.New("matches function requires both string and pattern arguments")
 	}
-	
+
 	// Visit the string expression
 	if err := con.visit(stringExpr); err != nil {
 		return err
 	}
-	
+
 	con.str.WriteString(" ~ ")
-	
+
 	// Visit the pattern expression and convert from RE2 to POSIX if it's a string literal
 	if constExpr := patternExpr.GetConstExpr(); constExpr != nil && constExpr.GetStringValue() != "" {
 		// Convert RE2 pattern to POSIX
@@ -427,7 +477,7 @@ func (con *converter) callMatches(target *exprpb.Expr, args []*exprpb.Expr) erro
 			return err
 		}
 	}
-	
+
 	return nil
 }
 
@@ -1442,45 +1492,45 @@ func isBinaryOrTernaryOperator(expr *exprpb.Expr) bool {
 // Note: This is a basic conversion for common patterns. Full RE2 to POSIX conversion is complex.
 func convertRE2ToPOSIX(re2Pattern string) string {
 	posixPattern := re2Pattern
-	
+
 	// Basic conversions for common differences between RE2 and POSIX:
-	
+
 	// 1. Word boundaries: \b -> [[:<:]] and [[:<:]] (PostgreSQL extension)
 	//    Note: PostgreSQL supports \y for word boundaries in some contexts
 	posixPattern = strings.ReplaceAll(posixPattern, `\b`, `\y`)
-	
+
 	// 2. Non-word boundaries: \B -> [^[:alnum:]_] (approximate)
 	//    This is a simplification; exact conversion is complex
 	posixPattern = strings.ReplaceAll(posixPattern, `\B`, `[^[:alnum:]_]`)
-	
+
 	// 3. Digit shortcuts: \d -> [[:digit:]] or [0-9]
 	posixPattern = strings.ReplaceAll(posixPattern, `\d`, `[[:digit:]]`)
-	
+
 	// 4. Non-digit shortcuts: \D -> [^[:digit:]] or [^0-9]
 	posixPattern = strings.ReplaceAll(posixPattern, `\D`, `[^[:digit:]]`)
-	
+
 	// 5. Word character shortcuts: \w -> [[:alnum:]_]
 	posixPattern = strings.ReplaceAll(posixPattern, `\w`, `[[:alnum:]_]`)
-	
+
 	// 6. Non-word character shortcuts: \W -> [^[:alnum:]_]
 	posixPattern = strings.ReplaceAll(posixPattern, `\W`, `[^[:alnum:]_]`)
-	
+
 	// 7. Whitespace shortcuts: \s -> [[:space:]]
 	posixPattern = strings.ReplaceAll(posixPattern, `\s`, `[[:space:]]`)
-	
+
 	// 8. Non-whitespace shortcuts: \S -> [^[:space:]]
 	posixPattern = strings.ReplaceAll(posixPattern, `\S`, `[^[:space:]]`)
-	
+
 	// Note: Many RE2 features are not directly convertible to POSIX ERE:
 	// - Lookahead/lookbehind assertions (?=...), (?!...), (?<=...), (?<!...)
 	// - Non-capturing groups (?:...)
 	// - Named groups (?P<name>...)
 	// - Case-insensitive flags (?i)
 	// - Multiline flags (?m)
 	// - Unicode character classes
-	// 
+	//
 	// For these cases, the pattern is returned as-is, which may cause PostgreSQL errors
 	// if the pattern uses unsupported RE2 features.
-	
+
 	return posixPattern
 }

go.mod

@@ -5,6 +5,7 @@ go 1.24
 require (
 	github.com/google/cel-go v0.26.0
 	github.com/jackc/pgx/v5 v5.7.5
+	github.com/lib/pq v1.10.9
 	github.com/spandigital/cel2sql v0.0.0-20250719023919-4359e01a942f
 	github.com/stretchr/testify v1.10.0
 	github.com/testcontainers/testcontainers-go v0.38.0