Skip to content

Commit 0e1950b

Browse files
sbdchdsbdchd
authored
parser: fix float lexing (#532)
- don't treat `cast` as a postfix expr, not sure what it was in there - add another alter table action - dedupe some parsing stuff
1 parent 061710b commit 0e1950b

File tree

228 files changed

+1385
-681
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

228 files changed

+1385
-681
lines changed

‎crates/squawk_lexer/src/lib.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,14 @@ impl Cursor<'_> {
302302
}
303303
_ => (),
304304
}
305+
} else {
306+
match self.first() {
307+
'e' | 'E' => {
308+
self.bump();
309+
empty_exponent = !self.eat_float_exponent();
310+
}
311+
_ => (),
312+
}
305313
}
306314
LiteralKind::Float {
307315
base,
@@ -623,6 +631,7 @@ $foo$hello$world$bar$
623631
1e-10
624632
1e+10
625633
1e10
634+
4664.E+5
626635
"#))
627636
}
628637

‎crates/squawk_lexer/src/snapshots/squawk_lexer__tests__floats.snap‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
source: crates/squawk_lexer/src/lib.rs
3+
expression: "lex(r#\"\n4664.E+5\n4664E+5\n4664E5\n4664e5\n \"#)"
4+
---
5+
[
6+
"\n" @ Whitespace,
7+
"4664.E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
8+
"\n" @ Whitespace,
9+
"4664E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
10+
"\n" @ Whitespace,
11+
"4664E5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
12+
"\n" @ Whitespace,
13+
"4664e5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
14+
"\n " @ Whitespace,
15+
]

‎crates/squawk_lexer/src/snapshots/squawk_lexer__tests__numeric.snap‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
22
source: crates/squawk_lexer/src/lib.rs
3-
expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n1e10\n\"#)"
3+
expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n1e10\n4664.E+5\n\"#)"
44
---
55
[
66
"\n" @ Whitespace,
@@ -24,4 +24,6 @@ expression: "lex(r#\"\n42\n3.5\n4.\n.001\n.123e10\n5e2\n1.925e-3\n1e-10\n1e+10\n
2424
"\n" @ Whitespace,
2525
"1e10" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
2626
"\n" @ Whitespace,
27+
"4664.E+5" @ Literal { kind: Float { base: Decimal, empty_exponent: false } },
28+
"\n" @ Whitespace,
2729
]

‎crates/squawk_parser/src/generated/syntax_kind.rs‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/squawk_parser/src/grammar.rs‎

Lines changed: 56 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,19 @@ fn atom_expr(p: &mut Parser<'_>) -> Option<CompletedMarker> {
789789
p.bump(POSITIONAL_PARAM);
790790
m.complete(p, LITERAL)
791791
}
792+
(CAST_KW | TREAT_KW, L_PAREN) => {
793+
let m = p.start();
794+
p.bump_any();
795+
p.bump(L_PAREN);
796+
if expr(p).is_none() {
797+
p.error("expected an expression");
798+
}
799+
p.expect(AS_KW);
800+
type_name(p);
801+
opt_collate(p);
802+
p.expect(R_PAREN);
803+
m.complete(p, CAST_EXPR)
804+
}
792805
(EXTRACT_KW, L_PAREN) => extract_fn(p),
793806
(JSON_EXISTS_KW, L_PAREN) => json_exists_fn(p),
794807
(JSON_ARRAY_KW, L_PAREN) => json_array_fn(p),
@@ -1260,20 +1273,6 @@ fn lhs(p: &mut Parser<'_>, r: &Restrictions) -> Option<CompletedMarker> {
12601273
p.bump_any();
12611274
(PREFIX_EXPR, 3)
12621275
}
1263-
CAST_KW | TREAT_KW => {
1264-
m = p.start();
1265-
p.bump_any();
1266-
p.expect(L_PAREN);
1267-
if expr(p).is_none() {
1268-
p.error("expected an expression");
1269-
}
1270-
p.expect(AS_KW);
1271-
type_name(p);
1272-
opt_collate(p);
1273-
p.expect(R_PAREN);
1274-
let cm = m.complete(p, CAST_EXPR);
1275-
return Some(cm);
1276-
}
12771276
OPERATOR_KW if p.at(OPERATOR_CALL) => {
12781277
m = p.start();
12791278
p.expect(OPERATOR_CALL);
@@ -1915,7 +1914,12 @@ fn call_expr_args(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
19151914
}
19161915
m.complete(p, OVER_CLAUSE);
19171916
}
1918-
m.complete(p, CALL_EXPR)
1917+
let cm = m.complete(p, CALL_EXPR);
1918+
if opt_string_literal(p).is_some() {
1919+
cm.precede(p).complete(p, CAST_EXPR)
1920+
} else {
1921+
cm
1922+
}
19191923
}
19201924

19211925
// foo[]
@@ -2017,13 +2021,13 @@ fn postfix_dot_expr<const FLOAT_RECOVERY: bool>(
20172021
if !FLOAT_RECOVERY {
20182022
assert!(p.at(DOT));
20192023
}
2020-
field_expr::<FLOAT_RECOVERY>(p, Some(lhs), allow_calls).map(|m| {
2024+
field_expr::<FLOAT_RECOVERY>(p, Some(lhs), allow_calls).map(|cm| {
20212025
// A field followed by a literal is a type cast so we insert a CAST_EXPR
20222026
// preceding it to wrap the previously parsed data.
2023-
if !p.at(NULL_KW) && !p.at(DEFAULT_KW) && literal(p).is_some() {
2024-
m.precede(p).complete(p, CAST_EXPR)
2027+
if opt_string_literal(p).is_some() {
2028+
cm.precede(p).complete(p, CAST_EXPR)
20252029
} else {
2026-
m
2030+
cm
20272031
}
20282032
})
20292033
}
@@ -4868,9 +4872,7 @@ fn savepoint(p: &mut Parser<'_>) -> CompletedMarker {
48684872
assert!(p.at(SAVEPOINT_KW));
48694873
let m = p.start();
48704874
p.bump(SAVEPOINT_KW);
4871-
if name_ref_(p).is_none() {
4872-
p.error("expected a name");
4873-
}
4875+
name_ref(p);
48744876
m.complete(p, SAVEPOINT)
48754877
}
48764878

@@ -4882,9 +4884,7 @@ fn release(p: &mut Parser<'_>) -> CompletedMarker {
48824884
let m = p.start();
48834885
p.bump(RELEASE_KW);
48844886
p.eat(SAVEPOINT_KW);
4885-
if name_ref_(p).is_none() {
4886-
p.error("expected a name");
4887-
}
4887+
name_ref(p);
48884888
m.complete(p, RELEASE_SAVEPOINT)
48894889
}
48904890

@@ -4909,9 +4909,7 @@ fn rollback(p: &mut Parser<'_>) -> CompletedMarker {
49094909
let _ = p.eat(WORK_KW) || p.eat(TRANSACTION_KW);
49104910
if is_rollback && p.eat(TO_KW) {
49114911
p.eat(SAVEPOINT_KW);
4912-
if name_ref_(p).is_none() {
4913-
p.error("expected a name");
4914-
}
4912+
name_ref(p);
49154913
} else if p.eat(AND_KW) {
49164914
p.eat(NO_KW);
49174915
p.expect(CHAIN_KW);
@@ -5275,15 +5273,7 @@ fn alter_server(p: &mut Parser<'_>) -> CompletedMarker {
52755273
string_literal(p);
52765274
found_option = true;
52775275
}
5278-
if p.eat(OPTIONS_KW) {
5279-
found_option = true;
5280-
p.expect(L_PAREN);
5281-
alter_option(p);
5282-
while !p.at(EOF) && p.eat(COMMA) {
5283-
alter_option(p);
5284-
}
5285-
p.expect(R_PAREN);
5286-
}
5276+
found_option |= opt_options_list(p);
52875277
if !found_option {
52885278
p.error("expected ALTER SERVER option");
52895279
}
@@ -7543,13 +7533,9 @@ fn alter_user_mapping(p: &mut Parser<'_>) -> CompletedMarker {
75437533
role(p);
75447534
p.expect(SERVER_KW);
75457535
name_ref(p);
7546-
p.expect(OPTIONS_KW);
7547-
p.expect(L_PAREN);
7548-
alter_option(p);
7549-
while !p.at(EOF) && p.eat(COMMA) {
7550-
alter_option(p);
7536+
if !opt_options_list(p) {
7537+
p.error("expected options");
75517538
}
7552-
p.expect(R_PAREN);
75537539
m.complete(p, ALTER_USER_MAPPING)
75547540
}
75557541

@@ -8337,13 +8323,9 @@ fn create_foreign_data_wrapper(p: &mut Parser<'_>) -> CompletedMarker {
83378323
fn opt_fdw_option(p: &mut Parser<'_>) -> bool {
83388324
match p.current() {
83398325
OPTIONS_KW => {
8340-
p.bump(OPTIONS_KW);
8341-
p.expect(L_PAREN);
8342-
alter_option(p);
8343-
while !p.at(EOF) && p.eat(COMMA) {
8344-
alter_option(p);
8326+
if !opt_options_list(p) {
8327+
p.error("expected options");
83458328
}
8346-
p.expect(R_PAREN);
83478329
true
83488330
}
83498331
HANDLER_KW | VALIDATOR_KW => {
@@ -8684,10 +8666,12 @@ fn publication_object(p: &mut Parser<'_>) {
86848666
if !p.eat(CURRENT_SCHEMA_KW) {
86858667
name_ref(p);
86868668
}
8687-
while !p.at(EOF) && p.eat(COMMA) {
8688-
if !p.eat(CURRENT_SCHEMA_KW) {
8689-
name_ref(p);
8669+
if p.eat(WHERE_KW) {
8670+
p.expect(L_PAREN);
8671+
if expr(p).is_none() {
8672+
p.error("expected expression");
86908673
}
8674+
p.expect(R_PAREN);
86918675
}
86928676
} else if p.eat(CURRENT_SCHEMA_KW) {
86938677
return;
@@ -9913,25 +9897,18 @@ fn explain_option(p: &mut Parser<'_>) {
99139897
}
99149898
}
99159899

9916-
// TODO: I think we want something like deliminated where we give it a FIRST
9917-
// token set so we can be robust to missing commas
9918-
fn one_or_more(p: &mut Parser<'_>, mut parse: impl FnMut(&mut Parser<'_>)) {
9919-
parse(p);
9920-
while !p.at(EOF) && p.eat(COMMA) {
9921-
parse(p);
9922-
}
9923-
}
9924-
99259900
// [ OPTIONS ( option 'value' [, ... ] ) ]
9926-
fn opt_options_list(p: &mut Parser<'_>) {
9927-
// [ OPTIONS ( option 'value' [, ... ] ) ]
9901+
fn opt_options_list(p: &mut Parser<'_>) -> bool {
99289902
if p.eat(OPTIONS_KW) {
99299903
p.expect(L_PAREN);
9930-
one_or_more(p, |p| {
9931-
col_label(p);
9932-
string_literal(p);
9933-
});
9904+
alter_option(p);
9905+
while !p.at(EOF) && p.eat(COMMA) {
9906+
alter_option(p);
9907+
}
99349908
p.expect(R_PAREN);
9909+
true
9910+
} else {
9911+
false
99359912
}
99369913
}
99379914

@@ -11900,15 +11877,9 @@ fn insert(p: &mut Parser<'_>, m: Option<Marker>) -> CompletedMarker {
1190011877
// ( { index_column_name | ( index_expression ) } [ COLLATE collation ] [ opclass ] [, ...] ) [ WHERE index_predicate ]
1190111878
} else if p.eat(L_PAREN) {
1190211879
while !p.at(EOF) {
11903-
// ( index_expression )
11904-
if p.eat(L_PAREN) {
11905-
// TODO: more strict?
11906-
if expr(p).is_none() {
11907-
p.error("expected index_expression");
11908-
}
11909-
// index_column_name
11880+
if expr(p).is_none() {
11881+
p.error("expected expression");
1191011882
}
11911-
name_ref(p);
1191211883
opt_collate(p);
1191311884
// [ opclass ]
1191411885
p.eat(IDENT);
@@ -13358,6 +13329,7 @@ fn alter_table_action(p: &mut Parser<'_>) -> Option<SyntaxKind> {
1335813329
// column_name
1335913330
name_ref(p);
1336013331
type_name(p);
13332+
opt_options_list(p);
1336113333
opt_collate(p);
1336213334
// [ column_constraint [ ... ] ]
1336313335
while !p.at(EOF) {
@@ -13373,7 +13345,7 @@ fn alter_table_action(p: &mut Parser<'_>) -> Option<SyntaxKind> {
1337313345
p.bump(ATTACH_KW);
1337413346
p.expect(PARTITION_KW);
1337513347
// name
13376-
name_ref(p);
13348+
path_name_ref(p);
1337713349
// { FOR VALUES partition_bound_spec | DEFAULT }
1337813350
partition_option(p);
1337913351
ATTACH_PARTITION
@@ -13521,6 +13493,12 @@ fn alter_table_action(p: &mut Parser<'_>) -> Option<SyntaxKind> {
1352113493
ALTER_COLUMN
1352213494
}
1352313495
}
13496+
OPTIONS_KW => {
13497+
if !opt_options_list(p) {
13498+
p.error("expected options list");
13499+
}
13500+
OPTIONS_LIST
13501+
}
1352413502
_ => return None,
1352513503
};
1352613504
Some(kind)
@@ -13759,13 +13737,9 @@ fn alter_column_option(p: &mut Parser<'_>) -> Option<SyntaxKind> {
1375913737
}
1376013738
// OPTIONS ( [ ADD | SET | DROP ] option ['value'] [, ... ])
1376113739
OPTIONS_KW => {
13762-
p.bump(OPTIONS_KW);
13763-
p.expect(L_PAREN);
13764-
alter_option(p);
13765-
while !p.at(EOF) && p.eat(COMMA) {
13766-
alter_option(p);
13740+
if !opt_options_list(p) {
13741+
p.error("expected options");
1376713742
}
13768-
p.expect(R_PAREN);
1376913743
SET_OPTIONS_LIST
1377013744
}
1377113745
// SET DEFAULT expression

‎crates/squawk_parser/src/lib.rs‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,6 +317,24 @@ impl<'t> Parser<'t> {
317317
m.complete(self, SyntaxKind::IS_NOT);
318318
return true;
319319
}
320+
// SyntaxKind::BYTE_STRING => {
321+
// let m = self.start();
322+
// self.bump(SyntaxKind::BYTE_STRING);
323+
// if self.eat(SyntaxKind::UESCAPE_KW) {
324+
// self.expect(SyntaxKind::STRING);
325+
// }
326+
// m.complete(self, SyntaxKind::BYTE_STRING);
327+
// return true;
328+
// }
329+
// SyntaxKind::IDENT => {
330+
// let m = self.start();
331+
// self.bump(SyntaxKind::IDENT);
332+
// if self.eat(SyntaxKind::UESCAPE_KW) {
333+
// self.expect(SyntaxKind::STRING);
334+
// }
335+
// m.complete(self, SyntaxKind::IDENT);
336+
// return true;
337+
// }
320338
SyntaxKind::CUSTOM_OP => {
321339
let m = self.start();
322340
while !self.at(SyntaxKind::EOF) {
@@ -412,7 +430,7 @@ impl<'t> Parser<'t> {
412430
if kind == SyntaxKind::EOF {
413431
return;
414432
}
415-
self.do_bump(kind, 1);
433+
self.bump(kind);
416434
}
417435

418436
/// Advances the parser by one token
@@ -568,6 +586,13 @@ impl<'t> Parser<'t> {
568586
SyntaxKind::LIKE_KW,
569587
TrivaBetween::Allowed,
570588
),
589+
// not ilike
590+
SyntaxKind::NOT_ILIKE => self.at_composite2(
591+
n,
592+
SyntaxKind::NOT_KW,
593+
SyntaxKind::ILIKE_KW,
594+
TrivaBetween::Allowed,
595+
),
571596
// not in
572597
SyntaxKind::NOT_IN => self.at_composite2(
573598
n,

0 commit comments

Comments
 (0)