Require space after -- to start single line comment in MySQL (apache#1705)

Author: hansott

src/dialect/mod.rs

@@ -881,6 +881,15 @@ pub trait Dialect: Debug + Any {
     fn supports_table_hints(&self) -> bool {
         false
     }
+
+    /// Returns true if this dialect requires a whitespace character after `--` to start a single line comment.
+    ///
+    /// MySQL: <https://dev.mysql.com/doc/refman/8.4/en/ansi-diff-comments.html>
+    /// e.g. UPDATE account SET balance=balance--1
+    //       WHERE account_id=5752             ^^^ will be interpreted as two minus signs instead of a comment
+    fn requires_single_line_comment_whitespace(&self) -> bool {
+        false
+    }
 }
 
 /// This represents the operators for which precedence must be defined

src/dialect/mysql.rs

@@ -125,6 +125,10 @@ impl Dialect for MySqlDialect {
     fn supports_table_hints(&self) -> bool {
         true
     }
+
+    fn requires_single_line_comment_whitespace(&self) -> bool {
+        true
+    }
 }
 
 /// `LOCK TABLES`

src/tokenizer.rs

@@ -1229,14 +1229,26 @@ impl<'a> Tokenizer<'a> {
                 // operators
                 '-' => {
                     chars.next(); // consume the '-'
+
                     match chars.peek() {
                         Some('-') => {
-                            chars.next(); // consume the second '-', starting a single-line comment
-                            let comment = self.tokenize_single_line_comment(chars);
-                            Ok(Some(Token::Whitespace(Whitespace::SingleLineComment {
-                                prefix: "--".to_owned(),
-                                comment,
-                            })))
+                            let mut is_comment = true;
+                            if self.dialect.requires_single_line_comment_whitespace() {
+                                is_comment = Some(' ') == chars.peekable.clone().nth(1);
+                            }
+
+                            if is_comment {
+                                chars.next(); // consume second '-'
+                                let comment = self.tokenize_single_line_comment(chars);
+                                return Ok(Some(Token::Whitespace(
+                                    Whitespace::SingleLineComment {
+                                        prefix: "--".to_owned(),
+                                        comment,
+                                    },
+                                )));
+                            }
+
+                            self.start_binop(chars, "-", Token::Minus)
                         }
                         Some('>') => {
                             chars.next();
@@ -3685,4 +3697,85 @@ mod tests {
             ],
         );
     }
+
+    #[test]
+    fn test_whitespace_required_after_single_line_comment() {
+        all_dialects_where(|dialect| dialect.requires_single_line_comment_whitespace())
+            .tokenizes_to(
+                "SELECT --'abc'",
+                vec![
+                    Token::make_keyword("SELECT"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::Minus,
+                    Token::Minus,
+                    Token::SingleQuotedString("abc".to_string()),
+                ],
+            );
+
+        all_dialects_where(|dialect| dialect.requires_single_line_comment_whitespace())
+            .tokenizes_to(
+                "SELECT -- 'abc'",
+                vec![
+                    Token::make_keyword("SELECT"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::Whitespace(Whitespace::SingleLineComment {
+                        prefix: "--".to_string(),
+                        comment: " 'abc'".to_string(),
+                    }),
+                ],
+            );
+
+        all_dialects_where(|dialect| dialect.requires_single_line_comment_whitespace())
+            .tokenizes_to(
+                "SELECT --",
+                vec![
+                    Token::make_keyword("SELECT"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::Minus,
+                    Token::Minus,
+                ],
+            );
+    }
+
+    #[test]
+    fn test_whitespace_not_required_after_single_line_comment() {
+        all_dialects_where(|dialect| !dialect.requires_single_line_comment_whitespace())
+            .tokenizes_to(
+                "SELECT --'abc'",
+                vec![
+                    Token::make_keyword("SELECT"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::Whitespace(Whitespace::SingleLineComment {
+                        prefix: "--".to_string(),
+                        comment: "'abc'".to_string(),
+                    }),
+                ],
+            );
+
+        all_dialects_where(|dialect| !dialect.requires_single_line_comment_whitespace())
+            .tokenizes_to(
+                "SELECT -- 'abc'",
+                vec![
+                    Token::make_keyword("SELECT"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::Whitespace(Whitespace::SingleLineComment {
+                        prefix: "--".to_string(),
+                        comment: " 'abc'".to_string(),
+                    }),
+                ],
+            );
+
+        all_dialects_where(|dialect| !dialect.requires_single_line_comment_whitespace())
+            .tokenizes_to(
+                "SELECT --",
+                vec![
+                    Token::make_keyword("SELECT"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::Whitespace(Whitespace::SingleLineComment {
+                        prefix: "--".to_string(),
+                        comment: "".to_string(),
+                    }),
+                ],
+            );
+    }
 }

tests/sqlparser_mysql.rs

@@ -3250,3 +3250,18 @@ fn parse_double_precision() {
         "CREATE TABLE foo (bar DOUBLE(11,0))",
     );
 }
+
+#[test]
+fn parse_looks_like_single_line_comment() {
+    mysql().one_statement_parses_to(
+        "UPDATE account SET balance=balance--1 WHERE account_id=5752",
+        "UPDATE account SET balance = balance - -1 WHERE account_id = 5752",
+    );
+    mysql().one_statement_parses_to(
+        r#"
+            UPDATE account SET balance=balance-- 1
+            WHERE account_id=5752
+        "#,
+        "UPDATE account SET balance = balance WHERE account_id = 5752",
+    );
+}