Skip to content

sqlparser: Remove unneeded escaping (#16255) #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Aug 8, 2025
Merged

sqlparser: Remove unneeded escaping (#16255) #175

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fae60c5
sqlparser: Remove unneeded escaping (#16255)
dbussink Jun 24, 2024
0bd5a8d
fix escape in tests
mhamza15 Aug 7, 2025
0a3b142
Merge branch 'release-20.0-github' into mhamza/backport-16255
mhamza15 Aug 7, 2025
afff925
fix runs on in workflows
mhamza15 Apr 2, 2025
793357a
Merge branch 'release-20.0-github' into mhamza/backport-16255
mhamza15 Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions go/sqltypes/value.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,7 +861,25 @@ var SQLEncodeMap [256]byte
// SQLDecodeMap is the reverse of SQLEncodeMap
var SQLDecodeMap [256]byte

// encodeRef is a map of characters we use for escaping.
// This doesn't include double quotes since we don't need
// to escape that, as we always generate single quoted strings.
var encodeRef = map[byte]byte{
'\x00': '0',
'\'': '\'',
'\b': 'b',
'\n': 'n',
'\r': 'r',
'\t': 't',
26: 'Z', // ctl-Z
'\\': '\\',
}

// decodeRef is a map of characters we use for unescaping.
// We do need all characters here, since we do accept
// escaped double quotes in single quote strings and
// double quoted strings.
var decodeRef = map[byte]byte{
'\x00': '0',
'\'': '\'',
'"': '"',
Expand Down Expand Up @@ -931,6 +949,11 @@ func init() {
for i := range SQLEncodeMap {
if to, ok := encodeRef[byte(i)]; ok {
SQLEncodeMap[byte(i)] = to
}
}

for i := range SQLDecodeMap {
if to, ok := decodeRef[byte(i)]; ok {
SQLDecodeMap[to] = byte(i)
}
}
Expand Down
8 changes: 4 additions & 4 deletions go/sqltypes/value_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ func TestEncode(t *testing.T) {
outASCII: "'Zm9v'",
}, {
in: TestValue(VarChar, "\x00'\"\b\n\r\t\x1A\\"),
outSQL: "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'",
outSQL: "'\\0\\'\"\\b\\n\\r\\t\\Z\\\\'",
outASCII: "'ACciCAoNCRpc'",
}, {
in: TestValue(Bit, "a"),
Expand Down Expand Up @@ -442,7 +442,7 @@ func TestEncodeStringSQL(t *testing.T) {
},
{
in: "\x00'\"\b\n\r\t\x1A\\",
out: "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'",
out: "'\\0\\'\"\\b\\n\\r\\t\\Z\\\\'",
},
}
for _, tcase := range testcases {
Expand Down Expand Up @@ -632,7 +632,7 @@ func TestEncodeSQLStringBuilder(t *testing.T) {
outSQL: "'foo'",
}, {
in: TestValue(VarChar, "\x00'\"\b\n\r\t\x1A\\"),
outSQL: "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'",
outSQL: "'\\0\\'\"\\b\\n\\r\\t\\Z\\\\'",
}, {
in: TestValue(Bit, "a"),
outSQL: "b'01100001'",
Expand Down Expand Up @@ -663,7 +663,7 @@ func TestEncodeSQLBytes2(t *testing.T) {
outSQL: "'foo'",
}, {
in: TestValue(VarChar, "\x00'\"\b\n\r\t\x1A\\"),
outSQL: "'\\0\\'\\\"\\b\\n\\r\\t\\Z\\\\'",
outSQL: "'\\0\\'\"\\b\\n\\r\\t\\Z\\\\'",
}, {
in: TestValue(Bit, "a"),
outSQL: "b'01100001'",
Expand Down
4 changes: 2 additions & 2 deletions go/vt/binlog/binlogplayer/binlog_player_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ func applyEvents(blp *BinlogPlayer) func() error {
func TestCreateVReplicationKeyRange(t *testing.T) {
want := "insert into _vt.vreplication " +
"(workflow, source, pos, max_tps, max_replication_lag, time_updated, transaction_timestamp, state, db_name, workflow_type, workflow_sub_type, defer_secondary_keys, options) " +
`values ('Resharding', 'keyspace:\"ks\" shard:\"0\" key_range:{end:\"\\x80\"}', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false, '{}')`
`values ('Resharding', 'keyspace:"ks" shard:"0" key_range:{end:"\\x80"}', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false, '{}')`

bls := binlogdatapb.BinlogSource{
Keyspace: "ks",
Expand All @@ -401,7 +401,7 @@ func TestCreateVReplicationKeyRange(t *testing.T) {
func TestCreateVReplicationTables(t *testing.T) {
want := "insert into _vt.vreplication " +
"(workflow, source, pos, max_tps, max_replication_lag, time_updated, transaction_timestamp, state, db_name, workflow_type, workflow_sub_type, defer_secondary_keys, options) " +
`values ('Resharding', 'keyspace:\"ks\" shard:\"0\" tables:\"a\" tables:\"b\"', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false, '{}')`
`values ('Resharding', 'keyspace:"ks" shard:"0" tables:"a" tables:"b"', 'MariaDB/0-1-1083', 9223372036854775807, 9223372036854775807, 481823, 0, 'Running', 'db', 0, 0, false, '{}')`

bls := binlogdatapb.BinlogSource{
Keyspace: "ks",
Expand Down
4 changes: 3 additions & 1 deletion go/vt/sqlparser/ast_format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -839,7 +839,9 @@ func (idx *IndexDefinition) Format(buf *TrackedBuffer) {
buf.astPrintf(idx, ")")

for _, opt := range idx.Options {
buf.astPrintf(idx, " %s", opt.Name)
if opt.Name != "" {
buf.astPrintf(idx, " %s", opt.Name)
}
if opt.String != "" {
buf.astPrintf(idx, " %#s", opt.String)
} else if opt.Value != nil {
Expand Down
170 changes: 88 additions & 82 deletions go/vt/sqlparser/parse_test.go
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions go/vt/sqlparser/testdata/select_cases.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4286,7 +4286,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"xt indexes"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"xt indexes\"' in boolean mode)
select * from t1 where match(a, b) against ('"xt indexes"' in boolean mode)
END
INPUT
select max(value) from t1 AS m LEFT JOIN t2 AS c1 ON m.c1id = c1.id AND c1.active = 'Yes' LEFT JOIN t3 AS c2 ON m.c2id = c2.id AND c2.active = 'Yes' WHERE m.pid=1 AND (c1.id IS NOT NULL OR c2.id IS NOT NULL);
Expand Down Expand Up @@ -5660,7 +5660,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text search" "now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text search\" \"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"text search" "now support"' in boolean mode)
END
INPUT
select insert('hello', 1, -18446744073709551616, 'hi');
Expand Down Expand Up @@ -8048,7 +8048,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text i"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text i\"' in boolean mode)
select * from t1 where match(a, b) against ('"text i"' in boolean mode)
END
INPUT
select column_name,data_type,CHARACTER_OCTET_LENGTH, CHARACTER_MAXIMUM_LENGTH from information_schema.columns where table_name='t1' order by column_name;
Expand Down Expand Up @@ -8144,7 +8144,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"support now"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"support now\"' in boolean mode)
select * from t1 where match(a, b) against ('"support now"' in boolean mode)
END
INPUT
select hex(inet_aton('127.1.1'));
Expand Down Expand Up @@ -12212,7 +12212,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text search" +"now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text search\" +\"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"text search" +"now support"' in boolean mode)
END
INPUT
select trigger_name from information_schema.triggers where event_object_table='t1';
Expand Down Expand Up @@ -14750,7 +14750,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST('"space model' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"space model' in boolean mode)
select * from t1 where match(a, b) against ('"space model' in boolean mode)
END
INPUT
select @ujis4 = CONVERT(@utf84 USING ujis);
Expand Down Expand Up @@ -14906,7 +14906,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"text search" -"now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"text search\" -\"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"text search" -"now support"' in boolean mode)
END
INPUT
select _rowid,t1._rowid,skey,sval from t1;
Expand Down Expand Up @@ -17612,7 +17612,7 @@ INPUT
select 'aaa','aa''a',"aa""a";
END
OUTPUT
select 'aaa', 'aa\'a', 'aa\"a' from dual
select 'aaa', 'aa\'a', 'aa"a' from dual
END
INPUT
select cast('18446744073709551615' as signed);
Expand Down Expand Up @@ -18542,7 +18542,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"now support"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"now support\"' in boolean mode)
select * from t1 where match(a, b) against ('"now support"' in boolean mode)
END
INPUT
select date_add("1997-12-31 23:59:59",INTERVAL "10000:99:99" HOUR_SECOND);
Expand Down Expand Up @@ -22148,7 +22148,7 @@ INPUT
select * from t1 where MATCH a,b AGAINST ('"Now sUPPort"' IN BOOLEAN MODE);
END
OUTPUT
select * from t1 where match(a, b) against ('\"Now sUPPort\"' in boolean mode)
select * from t1 where match(a, b) against ('"Now sUPPort"' in boolean mode)
END
INPUT
select sum(all a),count(all a),avg(all a),std(all a),variance(all a),bit_or(all a),bit_and(all a),min(all a),max(all a),min(all c),max(all c) from t1;
Expand Down
28 changes: 14 additions & 14 deletions go/vt/vtgate/planbuilder/testdata/select_cases.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3096,8 +3096,8 @@
"Name": "main",
"Sharded": false
},
"FieldQuery": "select json_quote('null'), json_quote('\\\"null\\\"'), json_object(BIN(1), 2, 'abc', ASCII(4)), json_array(1, 'abc', null, true, curtime()) from dual where 1 != 1",
"Query": "select json_quote('null'), json_quote('\\\"null\\\"'), json_object(BIN(1), 2, 'abc', ASCII(4)), json_array(1, 'abc', null, true, curtime()) from dual",
"FieldQuery": "select json_quote('null'), json_quote('\"null\"'), json_object(BIN(1), 2, 'abc', ASCII(4)), json_array(1, 'abc', null, true, curtime()) from dual where 1 != 1",
"Query": "select json_quote('null'), json_quote('\"null\"'), json_object(BIN(1), 2, 'abc', ASCII(4)), json_array(1, 'abc', null, true, curtime()) from dual",
"Table": "dual"
},
"TablesUsed": [
Expand Down Expand Up @@ -3197,8 +3197,8 @@
"Name": "main",
"Sharded": false
},
"FieldQuery": "select json_schema_valid('{\\\"type\\\":\\\"string\\\",\\\"pattern\\\":\\\"(\\\"}', '\\\"abc\\\"'), json_schema_validation_report('{\\\"type\\\":\\\"string\\\",\\\"pattern\\\":\\\"(\\\"}', '\\\"abc\\\"') from dual where 1 != 1",
"Query": "select json_schema_valid('{\\\"type\\\":\\\"string\\\",\\\"pattern\\\":\\\"(\\\"}', '\\\"abc\\\"'), json_schema_validation_report('{\\\"type\\\":\\\"string\\\",\\\"pattern\\\":\\\"(\\\"}', '\\\"abc\\\"') from dual",
"FieldQuery": "select json_schema_valid('{\"type\":\"string\",\"pattern\":\"(\"}', '\"abc\"'), json_schema_validation_report('{\"type\":\"string\",\"pattern\":\"(\"}', '\"abc\"') from dual where 1 != 1",
"Query": "select json_schema_valid('{\"type\":\"string\",\"pattern\":\"(\"}', '\"abc\"'), json_schema_validation_report('{\"type\":\"string\",\"pattern\":\"(\"}', '\"abc\"') from dual",
"Table": "dual"
},
"TablesUsed": [
Expand All @@ -3219,8 +3219,8 @@
"Name": "main",
"Sharded": false
},
"FieldQuery": "select json_contains('{\\\"a\\\": 1, \\\"b\\\": 2, \\\"c\\\": {\\\"d\\\": 4}}', '1'), json_contains_path('{\\\"a\\\": 1, \\\"b\\\": 2, \\\"c\\\": {\\\"d\\\": 4}}', 'one', '$.a', '$.e'), json_extract('[10, 20, [30, 40]]', '$[1]'), json_unquote(json_extract('[\\\"a\\\",\\\"b\\\"]', '$[1]')), json_keys('{\\\"a\\\": 1, \\\"b\\\": {\\\"c\\\": 30}}'), json_overlaps('[1,3,5,7]', '[2,5,7]'), json_search('[\\\"abc\\\"]', 'one', 'abc'), json_value('{\\\"fname\\\": \\\"Joe\\\", \\\"lname\\\": \\\"Palmer\\\"}', '$.fname'), json_array(4, 5) member of ('[[3,4],[4,5]]') from dual where 1 != 1",
"Query": "select json_contains('{\\\"a\\\": 1, \\\"b\\\": 2, \\\"c\\\": {\\\"d\\\": 4}}', '1'), json_contains_path('{\\\"a\\\": 1, \\\"b\\\": 2, \\\"c\\\": {\\\"d\\\": 4}}', 'one', '$.a', '$.e'), json_extract('[10, 20, [30, 40]]', '$[1]'), json_unquote(json_extract('[\\\"a\\\",\\\"b\\\"]', '$[1]')), json_keys('{\\\"a\\\": 1, \\\"b\\\": {\\\"c\\\": 30}}'), json_overlaps('[1,3,5,7]', '[2,5,7]'), json_search('[\\\"abc\\\"]', 'one', 'abc'), json_value('{\\\"fname\\\": \\\"Joe\\\", \\\"lname\\\": \\\"Palmer\\\"}', '$.fname'), json_array(4, 5) member of ('[[3,4],[4,5]]') from dual",
"FieldQuery": "select json_contains('{\"a\": 1, \"b\": 2, \"c\": {\"d\": 4}}', '1'), json_contains_path('{\"a\": 1, \"b\": 2, \"c\": {\"d\": 4}}', 'one', '$.a', '$.e'), json_extract('[10, 20, [30, 40]]', '$[1]'), json_unquote(json_extract('[\"a\",\"b\"]', '$[1]')), json_keys('{\"a\": 1, \"b\": {\"c\": 30}}'), json_overlaps('[1,3,5,7]', '[2,5,7]'), json_search('[\"abc\"]', 'one', 'abc'), json_value('{\"fname\": \"Joe\", \"lname\": \"Palmer\"}', '$.fname'), json_array(4, 5) member of ('[[3,4],[4,5]]') from dual where 1 != 1",
"Query": "select json_contains('{\"a\": 1, \"b\": 2, \"c\": {\"d\": 4}}', '1'), json_contains_path('{\"a\": 1, \"b\": 2, \"c\": {\"d\": 4}}', 'one', '$.a', '$.e'), json_extract('[10, 20, [30, 40]]', '$[1]'), json_unquote(json_extract('[\"a\",\"b\"]', '$[1]')), json_keys('{\"a\": 1, \"b\": {\"c\": 30}}'), json_overlaps('[1,3,5,7]', '[2,5,7]'), json_search('[\"abc\"]', 'one', 'abc'), json_value('{\"fname\": \"Joe\", \"lname\": \"Palmer\"}', '$.fname'), json_array(4, 5) member of ('[[3,4],[4,5]]') from dual",
"Table": "dual"
},
"TablesUsed": [
Expand Down Expand Up @@ -3285,8 +3285,8 @@
"Name": "main",
"Sharded": false
},
"FieldQuery": "select json_depth('{}'), json_length('{\\\"a\\\": 1, \\\"b\\\": {\\\"c\\\": 30}}', '$.b'), json_type(json_extract('{\\\"a\\\": [10, true]}', '$.a')), json_valid('{\\\"a\\\": 1}') from dual where 1 != 1",
"Query": "select json_depth('{}'), json_length('{\\\"a\\\": 1, \\\"b\\\": {\\\"c\\\": 30}}', '$.b'), json_type(json_extract('{\\\"a\\\": [10, true]}', '$.a')), json_valid('{\\\"a\\\": 1}') from dual",
"FieldQuery": "select json_depth('{}'), json_length('{\"a\": 1, \"b\": {\"c\": 30}}', '$.b'), json_type(json_extract('{\"a\": [10, true]}', '$.a')), json_valid('{\"a\": 1}') from dual where 1 != 1",
"Query": "select json_depth('{}'), json_length('{\"a\": 1, \"b\": {\"c\": 30}}', '$.b'), json_type(json_extract('{\"a\": [10, true]}', '$.a')), json_valid('{\"a\": 1}') from dual",
"Table": "dual"
},
"TablesUsed": [
Expand All @@ -3307,8 +3307,8 @@
"Name": "main",
"Sharded": false
},
"FieldQuery": "select json_array_append('{\\\"a\\\": 1}', '$', 'z'), json_array_insert('[\\\"a\\\", {\\\"b\\\": [1, 2]}, [3, 4]]', '$[0]', 'x', '$[2][1]', 'y'), json_insert('{ \\\"a\\\": 1, \\\"b\\\": [2, 3]}', '$.a', 10, '$.c', cast('[true, false]' as JSON)) from dual where 1 != 1",
"Query": "select json_array_append('{\\\"a\\\": 1}', '$', 'z'), json_array_insert('[\\\"a\\\", {\\\"b\\\": [1, 2]}, [3, 4]]', '$[0]', 'x', '$[2][1]', 'y'), json_insert('{ \\\"a\\\": 1, \\\"b\\\": [2, 3]}', '$.a', 10, '$.c', cast('[true, false]' as JSON)) from dual",
"FieldQuery": "select json_array_append('{\"a\": 1}', '$', 'z'), json_array_insert('[\"a\", {\"b\": [1, 2]}, [3, 4]]', '$[0]', 'x', '$[2][1]', 'y'), json_insert('{ \"a\": 1, \"b\": [2, 3]}', '$.a', 10, '$.c', cast('[true, false]' as JSON)) from dual where 1 != 1",
"Query": "select json_array_append('{\"a\": 1}', '$', 'z'), json_array_insert('[\"a\", {\"b\": [1, 2]}, [3, 4]]', '$[0]', 'x', '$[2][1]', 'y'), json_insert('{ \"a\": 1, \"b\": [2, 3]}', '$.a', 10, '$.c', cast('[true, false]' as JSON)) from dual",
"Table": "dual"
},
"TablesUsed": [
Expand All @@ -3329,8 +3329,8 @@
"Name": "main",
"Sharded": false
},
"FieldQuery": "select json_merge('[1, 2]', '[true, false]'), json_merge_patch('{\\\"name\\\": \\\"x\\\"}', '{\\\"id\\\": 47}'), json_merge_preserve('[1, 2]', '{\\\"id\\\": 47}') from dual where 1 != 1",
"Query": "select json_merge('[1, 2]', '[true, false]'), json_merge_patch('{\\\"name\\\": \\\"x\\\"}', '{\\\"id\\\": 47}'), json_merge_preserve('[1, 2]', '{\\\"id\\\": 47}') from dual",
"FieldQuery": "select json_merge('[1, 2]', '[true, false]'), json_merge_patch('{\"name\": \"x\"}', '{\"id\": 47}'), json_merge_preserve('[1, 2]', '{\"id\": 47}') from dual where 1 != 1",
"Query": "select json_merge('[1, 2]', '[true, false]'), json_merge_patch('{\"name\": \"x\"}', '{\"id\": 47}'), json_merge_preserve('[1, 2]', '{\"id\": 47}') from dual",
"Table": "dual"
},
"TablesUsed": [
Expand All @@ -3351,8 +3351,8 @@
"Name": "main",
"Sharded": false
},
"FieldQuery": "select json_remove('[1, [2, 3], 4]', '$[1]'), json_replace('{ \\\"a\\\": 1, \\\"b\\\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_set('{ \\\"a\\\": 1, \\\"b\\\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_unquote('\\\"abc\\\"') from dual where 1 != 1",
"Query": "select json_remove('[1, [2, 3], 4]', '$[1]'), json_replace('{ \\\"a\\\": 1, \\\"b\\\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_set('{ \\\"a\\\": 1, \\\"b\\\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_unquote('\\\"abc\\\"') from dual",
"FieldQuery": "select json_remove('[1, [2, 3], 4]', '$[1]'), json_replace('{ \"a\": 1, \"b\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_set('{ \"a\": 1, \"b\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_unquote('\"abc\"') from dual where 1 != 1",
"Query": "select json_remove('[1, [2, 3], 4]', '$[1]'), json_replace('{ \"a\": 1, \"b\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_set('{ \"a\": 1, \"b\": [2, 3]}', '$.a', 10, '$.c', '[true, false]'), json_unquote('\"abc\"') from dual",
"Table": "dual"
},
"TablesUsed": [
Expand Down
Loading
Loading