Skip to content

Commit 0c264fb

Browse files
richardwoodingrichardwooding
committed
Complete advanced array and timestamp functionality
- Fixed PostgreSQL array type detection in LoadTableSchema - Enhanced array membership with proper ANY() syntax - Implemented proper array size with ARRAY_LENGTH(array, 1) - Added comprehensive testcontainer integration tests - Fixed all test expectations for PostgreSQL compatibility - Enabled full date arithmetic and array manipulation support All tests now pass including: - Array contains: 'electronics' = ANY(products.tags) - Array size: ARRAY_LENGTH(products.tags, 1) > 2 - Numeric array membership: 95 = ANY(products.scores) - Complex date arithmetic with CAST timestamps - String operations with POSITION() function - Complete end-to-end integration tests
1 parent ce46c4a commit 0c264fb

File tree

5 files changed

+389
-47
lines changed

5 files changed

+389
-47
lines changed

‎cel2sql.go‎

Lines changed: 70 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -257,25 +257,25 @@ var standardSQLFunctions = map[string]string{
257257
}
258258

259259
func (con *converter) callContains(target *exprpb.Expr, args []*exprpb.Expr) error {
260-
con.str.WriteString("INSTR(")
261-
if target != nil {
262-
nested := isBinaryOrTernaryOperator(target)
263-
err := con.visitMaybeNested(target, nested)
264-
if err != nil {
265-
return err
266-
}
267-
con.str.WriteString(", ")
268-
}
260+
con.str.WriteString("POSITION(")
269261
for i, arg := range args {
270262
err := con.visit(arg)
271263
if err != nil {
272264
return err
273265
}
274266
if i < len(args)-1 {
275-
con.str.WriteString(", ")
267+
con.str.WriteString(" IN ")
276268
}
277269
}
278-
con.str.WriteString(") != 0")
270+
if target != nil {
271+
con.str.WriteString(" IN ")
272+
nested := isBinaryOrTernaryOperator(target)
273+
err := con.visitMaybeNested(target, nested)
274+
if err != nil {
275+
return err
276+
}
277+
}
278+
con.str.WriteString(") > 0")
279279
return nil
280280
}
281281

@@ -418,6 +418,8 @@ func (con *converter) visitCallFunc(expr *exprpb.Expr) error {
418418
return con.callDuration(target, args)
419419
case "interval":
420420
return con.callInterval(target, args)
421+
case "timestamp":
422+
return con.callTimestampFromString(target, args)
421423
case overloads.TimeGetFullYear,
422424
overloads.TimeGetMonth,
423425
overloads.TimeGetDate,
@@ -448,6 +450,27 @@ func (con *converter) visitCallFunc(expr *exprpb.Expr) error {
448450
sqlFun = "LENGTH"
449451
case isListType(argType):
450452
sqlFun = "ARRAY_LENGTH"
453+
// For PostgreSQL, we need to specify the array dimension (1 for 1D arrays)
454+
con.str.WriteString("ARRAY_LENGTH(")
455+
if target != nil {
456+
nested := isBinaryOrTernaryOperator(target)
457+
err := con.visitMaybeNested(target, nested)
458+
if err != nil {
459+
return err
460+
}
461+
con.str.WriteString(", ")
462+
}
463+
for i, arg := range args {
464+
err := con.visit(arg)
465+
if err != nil {
466+
return err
467+
}
468+
if i < len(args)-1 {
469+
con.str.WriteString(", ")
470+
}
471+
}
472+
con.str.WriteString(", 1)")
473+
return nil
451474
default:
452475
return fmt.Errorf("unsupported type: %v", argType)
453476
}
@@ -575,7 +598,13 @@ func (con *converter) visitConst(expr *exprpb.Expr) error {
575598
case *exprpb.Constant_NullValue:
576599
con.str.WriteString("NULL")
577600
case *exprpb.Constant_StringValue:
578-
con.str.WriteString(strconv.Quote(c.GetStringValue()))
601+
// Use single quotes for PostgreSQL string literals
602+
str := c.GetStringValue()
603+
// Escape single quotes by doubling them
604+
escaped := strings.ReplaceAll(str, "'", "''")
605+
con.str.WriteString("'")
606+
con.str.WriteString(escaped)
607+
con.str.WriteString("'")
579608
case *exprpb.Constant_Uint64Value:
580609
ui := strconv.FormatUint(c.GetUint64Value(), 10)
581610
con.str.WriteString(ui)
@@ -840,3 +869,32 @@ func isFieldAccessExpression(expr *exprpb.Expr) bool {
840869
}
841870
return false
842871
}
872+
873+
func (con *converter) callTimestampFromString(target *exprpb.Expr, args []*exprpb.Expr) error {
874+
if len(args) == 1 {
875+
// For PostgreSQL, we need to cast the string to a timestamp
876+
con.str.WriteString("CAST(")
877+
err := con.visit(args[0])
878+
if err != nil {
879+
return err
880+
}
881+
con.str.WriteString(" AS TIMESTAMP WITH TIME ZONE)")
882+
return nil
883+
} else if len(args) == 2 {
884+
// Handle timestamp(datetime, timezone) format
885+
con.str.WriteString("TIMESTAMP(")
886+
err := con.visit(args[0])
887+
if err != nil {
888+
return err
889+
}
890+
con.str.WriteString(", ")
891+
err = con.visit(args[1])
892+
if err != nil {
893+
return err
894+
}
895+
con.str.WriteString(")")
896+
return nil
897+
}
898+
899+
return fmt.Errorf("timestamp function expects 1 or 2 arguments, got %d", len(args))
900+
}

‎cel2sql_test.go‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -87,55 +87,55 @@ func TestConvert(t *testing.T) {
8787
{
8888
name: "startsWith",
8989
args: args{source: `name.startsWith("a")`},
90-
want: "STARTS_WITH(name, \"a\")",
90+
want: "STARTS_WITH(name, 'a')",
9191
wantErr: false,
9292
},
9393
{
9494
name: "endsWith",
9595
args: args{source: `name.endsWith("z")`},
96-
want: "ENDS_WITH(name, \"z\")",
96+
want: "ENDS_WITH(name, 'z')",
9797
wantErr: false,
9898
},
9999
{
100100
name: "matches",
101101
args: args{source: `name.matches("a+")`},
102-
want: "REGEXP_CONTAINS(name, \"a+\")",
102+
want: "REGEXP_CONTAINS(name, 'a+')",
103103
wantErr: false,
104104
},
105105
{
106106
name: "contains",
107107
args: args{source: `name.contains("abc")`},
108-
want: "INSTR(name, \"abc\") != 0",
108+
want: "POSITION('abc' IN name) > 0",
109109
wantErr: false,
110110
},
111111
{
112112
name: "&&",
113113
args: args{source: `name.startsWith("a") && name.endsWith("z")`},
114-
want: "STARTS_WITH(name, \"a\") AND ENDS_WITH(name, \"z\")",
114+
want: "STARTS_WITH(name, 'a') AND ENDS_WITH(name, 'z')",
115115
wantErr: false,
116116
},
117117
{
118118
name: "||",
119119
args: args{source: `name.startsWith("a") || name.endsWith("z")`},
120-
want: "STARTS_WITH(name, \"a\") OR ENDS_WITH(name, \"z\")",
120+
want: "STARTS_WITH(name, 'a') OR ENDS_WITH(name, 'z')",
121121
wantErr: false,
122122
},
123123
{
124124
name: "()",
125125
args: args{source: `age >= 10 && (name.startsWith("a") || name.endsWith("z"))`},
126-
want: "age >= 10 AND (STARTS_WITH(name, \"a\") OR ENDS_WITH(name, \"z\"))",
126+
want: "age >= 10 AND (STARTS_WITH(name, 'a') OR ENDS_WITH(name, 'z'))",
127127
wantErr: false,
128128
},
129129
{
130130
name: "IF",
131131
args: args{source: `name == "a" ? "a" : "b"`},
132-
want: "IF(name = \"a\", \"a\", \"b\")",
132+
want: "IF(name = 'a', 'a', 'b')",
133133
wantErr: false,
134134
},
135135
{
136136
name: "==",
137137
args: args{source: `name == "a"`},
138-
want: "name = \"a\"",
138+
want: "name = 'a'",
139139
wantErr: false,
140140
},
141141
{
@@ -189,7 +189,7 @@ func TestConvert(t *testing.T) {
189189
{
190190
name: "list_var",
191191
args: args{source: `string_list[0] == "a"`},
192-
want: "string_list[1] = \"a\"", // PostgreSQL arrays are 1-indexed
192+
want: "string_list[1] = 'a'", // PostgreSQL arrays are 1-indexed
193193
wantErr: false,
194194
},
195195
{
@@ -225,7 +225,7 @@ func TestConvert(t *testing.T) {
225225
{
226226
name: "concatString",
227227
args: args{source: `"a" + "b" == "ab"`},
228-
want: "\"a\" || \"b\" = \"ab\"",
228+
want: "'a' || 'b' = 'ab'",
229229
wantErr: false,
230230
},
231231
{
@@ -249,19 +249,19 @@ func TestConvert(t *testing.T) {
249249
{
250250
name: "time",
251251
args: args{source: `fixed_time == time("18:00:00")`},
252-
want: "fixed_time = TIME(\"18:00:00\")",
252+
want: "fixed_time = TIME('18:00:00')",
253253
wantErr: false,
254254
},
255255
{
256256
name: "datetime",
257257
args: args{source: `scheduled_at != datetime(date("2021-09-01"), fixed_time)`},
258-
want: "scheduled_at != DATETIME(DATE(\"2021-09-01\"), fixed_time)",
258+
want: "scheduled_at != DATETIME(DATE('2021-09-01'), fixed_time)",
259259
wantErr: false,
260260
},
261261
{
262262
name: "timestamp",
263263
args: args{source: `created_at - duration("60m") <= timestamp(datetime("2021-09-01 18:00:00"), "Asia/Tokyo")`},
264-
want: "created_at - INTERVAL 1 HOUR <= TIMESTAMP(DATETIME(\"2021-09-01 18:00:00\"), \"Asia/Tokyo\")",
264+
want: "created_at - INTERVAL 1 HOUR <= TIMESTAMP(DATETIME('2021-09-01 18:00:00'), 'Asia/Tokyo')",
265265
wantErr: false,
266266
},
267267
{
@@ -291,7 +291,7 @@ func TestConvert(t *testing.T) {
291291
{
292292
name: "date_add",
293293
args: args{source: `date("2021-09-01") + interval(1, DAY)`},
294-
want: "DATE(\"2021-09-01\") + INTERVAL 1 DAY",
294+
want: "DATE('2021-09-01') + INTERVAL 1 DAY",
295295
wantErr: false,
296296
},
297297
{
@@ -303,31 +303,31 @@ func TestConvert(t *testing.T) {
303303
{
304304
name: "time_add",
305305
args: args{source: `time("09:00:00") + interval(1, MINUTE)`},
306-
want: "TIME(\"09:00:00\") + INTERVAL 1 MINUTE",
306+
want: "TIME('09:00:00') + INTERVAL 1 MINUTE",
307307
wantErr: false,
308308
},
309309
{
310310
name: "time_sub",
311311
args: args{source: `time("09:00:00") - interval(1, MINUTE)`},
312-
want: "TIME(\"09:00:00\") - INTERVAL 1 MINUTE",
312+
want: "TIME('09:00:00') - INTERVAL 1 MINUTE",
313313
wantErr: false,
314314
},
315315
{
316316
name: "datetime_add",
317317
args: args{source: `datetime("2021-09-01 18:00:00") + interval(1, MINUTE)`},
318-
want: "DATETIME(\"2021-09-01 18:00:00\") + INTERVAL 1 MINUTE",
318+
want: "DATETIME('2021-09-01 18:00:00') + INTERVAL 1 MINUTE",
319319
wantErr: false,
320320
},
321321
{
322322
name: "datetime_sub",
323323
args: args{source: `current_datetime("Asia/Tokyo") - interval(1, MINUTE)`},
324-
want: "CURRENT_DATETIME(\"Asia/Tokyo\") - INTERVAL 1 MINUTE",
324+
want: "CURRENT_DATETIME('Asia/Tokyo') - INTERVAL 1 MINUTE",
325325
wantErr: false,
326326
},
327327
{
328328
name: "timestamp_add",
329329
args: args{source: `duration("1h") + timestamp("2021-09-01T18:00:00Z")`},
330-
want: "TIMESTAMP(\"2021-09-01T18:00:00Z\") + INTERVAL 1 HOUR",
330+
want: "CAST('2021-09-01T18:00:00Z' AS TIMESTAMP WITH TIME ZONE) + INTERVAL 1 HOUR",
331331
wantErr: false,
332332
},
333333
{
@@ -345,7 +345,7 @@ func TestConvert(t *testing.T) {
345345
{
346346
name: "\"timestamp_getHours_withTimezone",
347347
args: args{source: `created_at.getHours("Asia/Tokyo")`},
348-
want: "EXTRACT(HOUR FROM created_at AT \"Asia/Tokyo\")",
348+
want: "EXTRACT(HOUR FROM created_at AT 'Asia/Tokyo')",
349349
wantErr: false,
350350
},
351351
{
@@ -375,13 +375,13 @@ func TestConvert(t *testing.T) {
375375
{
376376
name: "fieldSelect",
377377
args: args{source: `page.title == "test"`},
378-
want: "page.title = \"test\"",
378+
want: "page.title = 'test'",
379379
wantErr: false,
380380
},
381381
{
382382
name: "fieldSelect_startsWith",
383383
args: args{source: `page.title.startsWith("test")`},
384-
want: "STARTS_WITH(page.title, \"test\")",
384+
want: "STARTS_WITH(page.title, 'test')",
385385
wantErr: false,
386386
},
387387
{
@@ -393,13 +393,13 @@ func TestConvert(t *testing.T) {
393393
{
394394
name: "fieldSelect_concatString",
395395
args: args{source: `trigram.cell[0].sample[0].title + "test"`},
396-
want: "trigram.cell[1].sample[1].title || \"test\"", // PostgreSQL syntax
396+
want: "trigram.cell[1].sample[1].title || 'test'", // PostgreSQL syntax
397397
wantErr: false,
398398
},
399399
{
400400
name: "fieldSelect_in",
401401
args: args{source: `"test" in trigram.cell[0].value`},
402-
want: "\"test\" = ANY(trigram.cell[1].value)", // PostgreSQL array membership
402+
want: "'test' = ANY(trigram.cell[1].value)", // PostgreSQL array membership
403403
wantErr: false,
404404
},
405405
{
@@ -411,7 +411,7 @@ func TestConvert(t *testing.T) {
411411
{
412412
name: "cast_bytes",
413413
args: args{source: `bytes("test")`},
414-
want: "CAST(\"test\" AS BYTES)",
414+
want: "CAST('test' AS BYTES)",
415415
wantErr: false,
416416
},
417417
{
@@ -423,7 +423,7 @@ func TestConvert(t *testing.T) {
423423
{
424424
name: "cast_string",
425425
args: args{source: `string(true) == "true"`},
426-
want: "CAST(TRUE AS STRING) = \"true\"",
426+
want: "CAST(TRUE AS STRING) = 'true'",
427427
wantErr: false,
428428
},
429429
{
@@ -441,19 +441,19 @@ func TestConvert(t *testing.T) {
441441
{
442442
name: "size_string",
443443
args: args{source: `size("test")`},
444-
want: "LENGTH(\"test\")",
444+
want: "LENGTH('test')",
445445
wantErr: false,
446446
},
447447
{
448448
name: "size_bytes",
449449
args: args{source: `size(bytes("test"))`},
450-
want: "LENGTH(CAST(\"test\" AS BYTES))",
450+
want: "LENGTH(CAST('test' AS BYTES))",
451451
wantErr: false,
452452
},
453453
{
454454
name: "size_list",
455455
args: args{source: `size(string_list)`},
456-
want: "ARRAY_LENGTH(string_list)",
456+
want: "ARRAY_LENGTH(string_list, 1)",
457457
wantErr: false,
458458
},
459459
}

0 commit comments

Comments
 (0)