Skip to content

Commit bebc159

Browse files
committed
Tokenize empty line comments correctly
MySQL likes to send "blocky" comments where a line comment is surrounded by empty comments, like so: ``` -- -- Table structure for table `foo` -- CREATE TABLE ... ``` Currently, these comments are tokenized as MINUS, MINUS instead of an empty line comment. This commits fixes this problem by accepting any kind of whitespace to delimit a line comment.
1 parent 44c4023 commit bebc159

File tree

1 file changed

+23
-1
lines changed

1 file changed

+23
-1
lines changed

src/tokenizer.rs

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1387,7 +1387,11 @@ impl<'a> Tokenizer<'a> {
13871387
Some('-') => {
13881388
let mut is_comment = true;
13891389
if self.dialect.requires_single_line_comment_whitespace() {
1390-
is_comment = Some(' ') == chars.peekable.clone().nth(1);
1390+
is_comment = chars
1391+
.peekable
1392+
.clone()
1393+
.nth(1)
1394+
.is_some_and(char::is_whitespace);
13911395
}
13921396

13931397
if is_comment {
@@ -4069,6 +4073,24 @@ mod tests {
40694073
Token::Minus,
40704074
],
40714075
);
4076+
4077+
all_dialects_where(|d| d.requires_single_line_comment_whitespace()).tokenizes_to(
4078+
"--\n-- Table structure for table...\n--\n",
4079+
vec![
4080+
Token::Whitespace(Whitespace::SingleLineComment {
4081+
prefix: "--".to_string(),
4082+
comment: "\n".to_string(),
4083+
}),
4084+
Token::Whitespace(Whitespace::SingleLineComment {
4085+
prefix: "--".to_string(),
4086+
comment: " Table structure for table...\n".to_string(),
4087+
}),
4088+
Token::Whitespace(Whitespace::SingleLineComment {
4089+
prefix: "--".to_string(),
4090+
comment: "\n".to_string(),
4091+
}),
4092+
],
4093+
);
40724094
}
40734095

40744096
#[test]

0 commit comments

Comments
 (0)