feat(query): Enhance JSON parsing support extended json5 syntax (#18550)

Author: b41sh

Cargo.lock

@@ -359,7 +359,7 @@ jaq-interpret = "1.5.0"
 jaq-parse = "1.0.3"
 jaq-std = "1.6.0"
 jiff = { version = "0.2.10", features = ["serde", "tzdb-bundle-always"] }
-jsonb = "0.5.3"
+jsonb = "0.5.4"
 jwt-simple = { version = "0.12.10", default-features = false, features = ["pure-rust"] }
 lenient_semver = "0.4.2"
 levenshtein_automata = "0.2.1"

Cargo.toml

@@ -7,14 +7,6 @@ output domain  : {NULL}
 output         : NULL
 
 
-error: 
-  --> SQL:1:1
-  |
-1 | parse_json('nuLL')
-  | ^^^^^^^^^^^^^^^^^^ expected ident, pos 3 while evaluating function `parse_json('nuLL')` in expr `CAST('nuLL' AS Variant)`
-
-
-
 ast            : parse_json('null')
 raw expr       : parse_json('null')
 checked expr   : CAST<String>("null" AS Variant)
@@ -24,14 +16,6 @@ output domain  : Undefined
 output         : 'null'
 
 
-error: 
-  --> SQL:1:1
-  |
-1 | parse_json('  ')
-  | ^^^^^^^^^^^^^^^^ EOF while parsing a value, pos 2 while evaluating function `parse_json('  ')` in expr `CAST('  ' AS Variant)`
-
-
-
 ast            : parse_json('true')
 raw expr       : parse_json('true')
 checked expr   : CAST<String>("true" AS Variant)
@@ -146,6 +130,24 @@ evaluation (internal):
 +--------+------------------------------------------------------------------------------------------------------------------------------------+
 
 
+ast            : parse_json('  ')
+raw expr       : parse_json('  ')
+checked expr   : CAST<String>("  " AS Variant)
+optimized expr : 0x2000000000000000
+output type    : Variant
+output domain  : Undefined
+output         : 'null'
+
+
+ast            : parse_json('nuLL')
+raw expr       : parse_json('nuLL')
+checked expr   : CAST<String>("nuLL" AS Variant)
+optimized expr : 0x2000000000000000
+output type    : Variant
+output domain  : Undefined
+output         : 'null'
+
+
 ast            : parse_json('+10')
 raw expr       : parse_json('+10')
 checked expr   : CAST<String>("+10" AS Variant)
@@ -182,6 +184,24 @@ output domain  : Undefined
 output         : '12'
 
 
+ast            : parse_json('0xabc')
+raw expr       : parse_json('0xabc')
+checked expr   : CAST<String>("0xabc" AS Variant)
+optimized expr : 0x2000000020000003500abc
+output type    : Variant
+output domain  : Undefined
+output         : '2748'
+
+
+ast            : parse_json('0x12abc.def')
+raw expr       : parse_json('0x12abc.def')
+checked expr   : CAST<String>("0x12abc.def" AS Variant)
+optimized expr : 0x20000000200000096040f2abcdef000000
+output type    : Variant
+output domain  : Undefined
+output         : '76476.87084960938'
+
+
 ast            : parse_json('99999999999999999999999999999999999999')
 raw expr       : parse_json('99999999999999999999999999999999999999')
 checked expr   : CAST<String>("99999999999999999999999999999999999999" AS Variant)
@@ -191,6 +211,15 @@ output domain  : Undefined
 output         : '99999999999999999999999999999999999999'
 
 
+ast            : parse_json('\'single quoted string\'')
+raw expr       : parse_json('\'single quoted string\'')
+checked expr   : CAST<String>("'single quoted string'" AS Variant)
+optimized expr : 0x200000001000001473696e676c652071756f74656420737472696e67
+output type    : Variant
+output domain  : Undefined
+output         : '"single quoted string"'
+
+
 ast            : parse_json('[1,2,,4]')
 raw expr       : parse_json('[1,2,,4]')
 checked expr   : CAST<String>("[1,2,,4]" AS Variant)
@@ -200,6 +229,23 @@ output domain  : Undefined
 output         : '[1,2,null,4]'
 
 
+ast            : parse_json('{ key :"val", key123_$测试 :"val" }')
+raw expr       : parse_json('{ key :"val", key123_$测试 :"val" }')
+checked expr   : CAST<String>("{ key :\"val\", key123_$测试 :\"val\" }" AS Variant)
+optimized expr : 0x40000002100000031000000e10000003100000036b65796b65793132335f24e6b58be8af9576616c76616c
+output type    : Variant
+output domain  : Undefined
+output         : '{"key":"val","key123_$测试":"val"}'
+
+
+error: 
+  --> SQL:1:1
+  |
+1 | parse_json('{ 123 :"val" }')
+  | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ object attribute name cannot be a number, pos 3 while evaluating function `parse_json('{ 123 :"val" }')` in expr `CAST('{ 123 :"val" }' AS Variant)`
+
+
+
 ast            : try_parse_json(NULL)
 raw expr       : try_parse_json(NULL)
 checked expr   : try_parse_json<Variant NULL>(CAST<NULL>(NULL AS Variant NULL))
@@ -212,10 +258,10 @@ output         : NULL
 ast            : try_parse_json('nuLL')
 raw expr       : try_parse_json('nuLL')
 checked expr   : try_parse_json<String>("nuLL")
-optimized expr : NULL
+optimized expr : 0x2000000000000000
 output type    : Variant NULL
-output domain  : {NULL}
-output         : NULL
+output domain  : Undefined
+output         : 'null'
 
 
 ast            : try_parse_json('null')
@@ -363,10 +409,10 @@ output         : NULL
 ast            : check_json('nuLL')
 raw expr       : check_json('nuLL')
 checked expr   : check_json<String>("nuLL")
-optimized expr : "expected ident, pos 3"
+optimized expr : NULL
 output type    : String NULL
-output domain  : {"expected ident, pos 3"..="expected ident, pos 3"}
-output         : 'expected ident, pos 3'
+output domain  : {NULL}
+output         : NULL
 
 
 ast            : check_json(s)

src/query/functions/tests/it/scalars/testdata/variant.txt

@@ -89,9 +89,7 @@ fn test_variant() {
 
 fn test_parse_json(file: &mut impl Write) {
     run_ast(file, "parse_json(NULL)", &[]);
-    run_ast(file, "parse_json('nuLL')", &[]);
     run_ast(file, "parse_json('null')", &[]);
-    run_ast(file, "parse_json('  ')", &[]);
     run_ast(file, "parse_json('true')", &[]);
     run_ast(file, "parse_json('false')", &[]);
     run_ast(file, "parse_json('\"测试\"')", &[]);
@@ -124,16 +122,27 @@ fn test_parse_json(file: &mut impl Write) {
     )]);
 
     // json extension syntax
+    run_ast(file, "parse_json('  ')", &[]);
+    run_ast(file, "parse_json('nuLL')", &[]);
     run_ast(file, "parse_json('+10')", &[]);
     run_ast(file, "parse_json('001')", &[]);
     run_ast(file, "parse_json('.12')", &[]);
     run_ast(file, "parse_json('12.')", &[]);
+    run_ast(file, "parse_json('0xabc')", &[]);
+    run_ast(file, "parse_json('0x12abc.def')", &[]);
     run_ast(
         file,
         "parse_json('99999999999999999999999999999999999999')",
         &[],
     );
+    run_ast(file, r#"parse_json('\'single quoted string\'')"#, &[]);
     run_ast(file, "parse_json('[1,2,,4]')", &[]);
+    run_ast(
+        file,
+        "parse_json('{ key :\"val\", key123_$测试 :\"val\" }')",
+        &[],
+    );
+    run_ast(file, "parse_json('{ 123 :\"val\" }')", &[]);
 }
 
 fn test_try_parse_json(file: &mut impl Write) {

src/query/functions/tests/it/scalars/variant.rs

@@ -84,6 +84,26 @@ select parse_json('[1,2, ,4]'), parse_json('[,2]'), parse_json('[1,]')
 ----
 [1,2,null,4] [null,2] [1,null]
 
+query TTTTT
+select parse_json('  '), parse_json('NULL'), parse_json('True'), parse_json('nan'), parse_json('+infinity')
+----
+null null true null null
+
+query TTT
+select parse_json('0x123'), parse_json('0XABCDEF'), parse_json('0XABC.DEF')
+----
+291 11259375 2748.870849609375
+
+query TT
+select parse_json('\'abc\''), parse_json('\'测试\'')
+----
+"abc" "测试"
+
+query TT
+select parse_json('{key:"val"}'), parse_json('{_中文_$key123:"val"}')
+----
+{"key":"val"} {"_中文_$key123":"val"}
+
 statement error 1006
 select parse_json('[1,')
 
@@ -242,3 +262,4 @@ select to_variant(a), to_variant(b), to_variant(c) from t3
 statement ok
 DROP DATABASE db1
 
+

tests/sqllogictests/suites/query/functions/02_0048_function_semi_structureds_parse_json.test

@@ -17,7 +17,7 @@ create table iv(a int not null, b variant not null)
 query TIITI
 copy into iv from @data/csv/invalid_variant.csv FILE_FORMAT = (field_delimiter = '\t' record_delimiter = '\n' type = CSV) disable_variant_check = false ON_ERROR = CONTINUE
 ----
-csv/invalid_variant.csv 1 1 Invalid value 'invalidvariant' for column 1 (b Variant): expected value, pos 1 1
+csv/invalid_variant.csv 1 1 Invalid value 'invalidvariant' for column 1 (b Variant): expected ident, pos 3 1
 
 query IT
 select * from iv