Merge branch 'main' into mysql-index-hints

AvivDavid-Satori · web-flow · commit af42d1802ed9 · 2025-01-26T10:51:29.000+02:00
diff --git a/src/dialect/bigquery.rs b/src/dialect/bigquery.rs
@@ -31,6 +31,11 @@ impl Dialect for BigQueryDialect {
         true
     }
 
+    /// See <https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement>
+    fn supports_column_definition_trailing_commas(&self) -> bool {
+        true
+    }
+
     fn is_identifier_start(&self, ch: char) -> bool {
         ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_'
     }
diff --git a/src/dialect/generic.rs b/src/dialect/generic.rs
@@ -139,4 +139,8 @@ impl Dialect for GenericDialect {
     fn supports_user_host_grantee(&self) -> bool {
         true
     }
+
+    fn supports_string_escape_constant(&self) -> bool {
+        true
+    }
 }
diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs
@@ -405,11 +405,18 @@ pub trait Dialect: Debug + Any {
     }
 
     /// Returns true if the dialect supports trailing commas in the `FROM` clause of a `SELECT` statement.
-    /// /// Example: `SELECT 1 FROM T, U, LIMIT 1`
+    /// Example: `SELECT 1 FROM T, U, LIMIT 1`
     fn supports_from_trailing_commas(&self) -> bool {
         false
     }
 
+    /// Returns true if the dialect supports trailing commas in the
+    /// column definitions list of a `CREATE` statement.
+    /// Example: `CREATE TABLE T (x INT, y TEXT,)`
+    fn supports_column_definition_trailing_commas(&self) -> bool {
+        false
+    }
+
     /// Returns true if the dialect supports double dot notation for object names
     ///
     /// Example
@@ -843,6 +850,12 @@ pub trait Dialect: Debug + Any {
 
     /// Returns true if the dialect supports the table hints in the `FROM` clause.
     fn supports_table_hints(&self) -> bool {
+      flase
+  }
+    /// Returns true if this dialect supports the E'...' syntax for string literals
+    ///
+    /// Postgres: <https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-STRINGS-ESCAPE>
+    fn supports_string_escape_constant(&self) -> bool {
         false
     }
 }
diff --git a/src/dialect/postgresql.rs b/src/dialect/postgresql.rs
@@ -245,6 +245,10 @@ impl Dialect for PostgreSqlDialect {
     fn supports_nested_comments(&self) -> bool {
         true
     }
+
+    fn supports_string_escape_constant(&self) -> bool {
+        true
+    }
 }
 
 pub fn parse_create(parser: &mut Parser) -> Option<Result<Statement, ParserError>> {
diff --git a/src/dialect/redshift.rs b/src/dialect/redshift.rs
@@ -109,4 +109,8 @@ impl Dialect for RedshiftSqlDialect {
     fn supports_partiql(&self) -> bool {
         true
     }
+
+    fn supports_string_escape_constant(&self) -> bool {
+        true
+    }
 }
diff --git a/src/parser/mod.rs b/src/parser/mod.rs
@@ -6718,7 +6718,11 @@ impl<'a> Parser<'a> {
                 return self.expected("',' or ')' after column definition", self.peek_token());
             };
 
-            if rparen && (!comma || self.options.trailing_commas) {
+            if rparen
+                && (!comma
+                    || self.dialect.supports_column_definition_trailing_commas()
+                    || self.options.trailing_commas)
+            {
                 let _ = self.consume_token(&Token::RParen);
                 break;
             }
@@ -9356,7 +9360,11 @@ impl<'a> Parser<'a> {
                 self.next_token();
                 Ok(vec![])
             } else {
-                let cols = self.parse_comma_separated(Parser::parse_view_column)?;
+                let cols = self.parse_comma_separated_with_trailing_commas(
+                    Parser::parse_view_column,
+                    self.dialect.supports_column_definition_trailing_commas(),
+                    None,
+                )?;
                 self.expect_token(&Token::RParen)?;
                 Ok(cols)
             }
diff --git a/src/test_utils.rs b/src/test_utils.rs
@@ -33,7 +33,7 @@ use core::fmt::Debug;
 
 use crate::dialect::*;
 use crate::parser::{Parser, ParserError};
-use crate::tokenizer::Tokenizer;
+use crate::tokenizer::{Token, Tokenizer};
 use crate::{ast::*, parser::ParserOptions};
 
 #[cfg(test)]
@@ -237,6 +237,22 @@ impl TestedDialects {
     pub fn verified_expr(&self, sql: &str) -> Expr {
         self.expr_parses_to(sql, sql)
     }
+
+    /// Check that the tokenizer returns the expected tokens for the given SQL.
+    pub fn tokenizes_to(&self, sql: &str, expected: Vec<Token>) {
+        if self.dialects.is_empty() {
+            panic!("No dialects to test");
+        }
+
+        self.dialects.iter().for_each(|dialect| {
+            let mut tokenizer = Tokenizer::new(&**dialect, sql);
+            if let Some(options) = &self.options {
+                tokenizer = tokenizer.with_unescape(options.unescape);
+            }
+            let tokens = tokenizer.tokenize().unwrap();
+            assert_eq!(expected, tokens, "Tokenized differently for {:?}", dialect);
+        });
+    }
 }
 
 /// Returns all available dialects.
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -971,7 +971,10 @@ impl<'a> Tokenizer<'a> {
                     match chars.peek() {
                         Some('\'') => {
                             // N'...' - a <national character string literal>
-                            let s = self.tokenize_single_quoted_string(chars, '\'', true)?;
+                            let backslash_escape =
+                                self.dialect.supports_string_literal_backslash_escape();
+                            let s =
+                                self.tokenize_single_quoted_string(chars, '\'', backslash_escape)?;
                             Ok(Some(Token::NationalStringLiteral(s)))
                         }
                         _ => {
@@ -982,7 +985,7 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 // PostgreSQL accepts "escape" string constants, which are an extension to the SQL standard.
-                x @ 'e' | x @ 'E' => {
+                x @ 'e' | x @ 'E' if self.dialect.supports_string_escape_constant() => {
                     let starting_loc = chars.location();
                     chars.next(); // consume, to check the next char
                     match chars.peek() {
@@ -2155,6 +2158,7 @@ mod tests {
     use crate::dialect::{
         BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect,
     };
+    use crate::test_utils::all_dialects_where;
     use core::fmt::Debug;
 
     #[test]
@@ -3543,4 +3547,74 @@ mod tests {
         ];
         compare(expected, tokens);
     }
+
+    #[test]
+    fn test_national_strings_backslash_escape_not_supported() {
+        all_dialects_where(|dialect| !dialect.supports_string_literal_backslash_escape())
+            .tokenizes_to(
+                "select n'''''\\'",
+                vec![
+                    Token::make_keyword("select"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::NationalStringLiteral("''\\".to_string()),
+                ],
+            );
+    }
+
+    #[test]
+    fn test_national_strings_backslash_escape_supported() {
+        all_dialects_where(|dialect| dialect.supports_string_literal_backslash_escape())
+            .tokenizes_to(
+                "select n'''''\\''",
+                vec![
+                    Token::make_keyword("select"),
+                    Token::Whitespace(Whitespace::Space),
+                    Token::NationalStringLiteral("'''".to_string()),
+                ],
+            );
+    }
+
+    #[test]
+    fn test_string_escape_constant_not_supported() {
+        all_dialects_where(|dialect| !dialect.supports_string_escape_constant()).tokenizes_to(
+            "select e'...'",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::make_word("e", None),
+                Token::SingleQuotedString("...".to_string()),
+            ],
+        );
+
+        all_dialects_where(|dialect| !dialect.supports_string_escape_constant()).tokenizes_to(
+            "select E'...'",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::make_word("E", None),
+                Token::SingleQuotedString("...".to_string()),
+            ],
+        );
+    }
+
+    #[test]
+    fn test_string_escape_constant_supported() {
+        all_dialects_where(|dialect| dialect.supports_string_escape_constant()).tokenizes_to(
+            "select e'\\''",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::EscapedStringLiteral("'".to_string()),
+            ],
+        );
+
+        all_dialects_where(|dialect| dialect.supports_string_escape_constant()).tokenizes_to(
+            "select E'\\''",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::EscapedStringLiteral("'".to_string()),
+            ],
+        );
+    }
 }
diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs
@@ -10265,15 +10265,19 @@ fn parse_trailing_comma() {
             "Expected: column name or constraint definition, found: )".to_string()
         )
     );
+
+    let unsupported_dialects = all_dialects_where(|d| !d.supports_trailing_commas());
+    assert_eq!(
+        unsupported_dialects
+            .parse_sql_statements("SELECT * FROM track ORDER BY milliseconds,")
+            .unwrap_err(),
+        ParserError::ParserError("Expected: an expression, found: EOF".to_string())
+    );
 }
 
 #[test]
 fn parse_projection_trailing_comma() {
-    // Some dialects allow trailing commas only in the projection
-    let trailing_commas = TestedDialects::new(vec![
-        Box::new(SnowflakeDialect {}),
-        Box::new(BigQueryDialect {}),
-    ]);
+    let trailing_commas = all_dialects_where(|d| d.supports_projection_trailing_commas());
 
     trailing_commas.one_statement_parses_to(
         "SELECT album_id, name, FROM track",
@@ -10286,20 +10290,14 @@ fn parse_projection_trailing_comma() {
 
     trailing_commas.verified_stmt("SELECT DISTINCT ON (album_id) name FROM track");
 
+    let unsupported_dialects = all_dialects_where(|d| {
+        !d.supports_projection_trailing_commas() && !d.supports_trailing_commas()
+    });
     assert_eq!(
-        trailing_commas
-            .parse_sql_statements("SELECT * FROM track ORDER BY milliseconds,")
+        unsupported_dialects
+            .parse_sql_statements("SELECT album_id, name, FROM track")
             .unwrap_err(),
-        ParserError::ParserError("Expected: an expression, found: EOF".to_string())
-    );
-
-    assert_eq!(
-        trailing_commas
-            .parse_sql_statements("CREATE TABLE employees (name text, age int,)")
-            .unwrap_err(),
-        ParserError::ParserError(
-            "Expected: column name or constraint definition, found: )".to_string()
-        ),
+        ParserError::ParserError("Expected an expression, found: FROM".to_string())
     );
 }
 
@@ -13137,6 +13135,33 @@ fn parse_overlaps() {
     verified_stmt("SELECT (DATE '2016-01-10', DATE '2016-02-01') OVERLAPS (DATE '2016-01-20', DATE '2016-02-10')");
 }
 
+#[test]
+fn parse_column_definition_trailing_commas() {
+    let dialects = all_dialects_where(|d| d.supports_column_definition_trailing_commas());
+
+    dialects.one_statement_parses_to("CREATE TABLE T (x INT64,)", "CREATE TABLE T (x INT64)");
+    dialects.one_statement_parses_to(
+        "CREATE TABLE T (x INT64, y INT64, )",
+        "CREATE TABLE T (x INT64, y INT64)",
+    );
+    dialects.one_statement_parses_to(
+        "CREATE VIEW T (x, y, ) AS SELECT 1",
+        "CREATE VIEW T (x, y) AS SELECT 1",
+    );
+
+    let unsupported_dialects = all_dialects_where(|d| {
+        !d.supports_projection_trailing_commas() && !d.supports_trailing_commas()
+    });
+    assert_eq!(
+        unsupported_dialects
+            .parse_sql_statements("CREATE TABLE employees (name text, age int,)")
+            .unwrap_err(),
+        ParserError::ParserError(
+            "Expected: column name or constraint definition, found: )".to_string()
+        ),
+    );
+}
+
 #[test]
 fn test_trailing_commas_in_from() {
     let dialects = all_dialects_where(|d| d.supports_from_trailing_commas());

Original file line number	Diff line number	Diff line change
`@@ -31,6 +31,11 @@ impl Dialect for BigQueryDialect {`
`31`	`31`	`true`
`32`	`32`	`}`
`33`	`33`
	`34`	`+ /// See <https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_table_statement>`
	`35`	`+ fn supports_column_definition_trailing_commas(&self) -> bool {`
	`36`	`+ true`
	`37`	`+ }`
	`38`	`+`
`34`	`39`	`fn is_identifier_start(&self, ch: char) -> bool {`
`35`	`40`	`ch.is_ascii_lowercase() \|\| ch.is_ascii_uppercase() \|\| ch == '_'`
`36`	`41`	`}`
Original file line number	Diff line number	Diff line change
`@@ -139,4 +139,8 @@ impl Dialect for GenericDialect {`
`139`	`139`	`fn supports_user_host_grantee(&self) -> bool {`
`140`	`140`	`true`
`141`	`141`	`}`
	`142`	`+`
	`143`	`+ fn supports_string_escape_constant(&self) -> bool {`
	`144`	`+ true`
	`145`	`+ }`
`142`	`146`	`}`
Original file line number	Diff line number	Diff line change
`@@ -245,6 +245,10 @@ impl Dialect for PostgreSqlDialect {`
`245`	`245`	`fn supports_nested_comments(&self) -> bool {`
`246`	`246`	`true`
`247`	`247`	`}`
	`248`	`+`
	`249`	`+ fn supports_string_escape_constant(&self) -> bool {`
	`250`	`+ true`
	`251`	`+ }`
`248`	`252`	`}`
`249`	`253`
`250`	`254`	`pub fn parse_create(parser: &mut Parser) -> Option<Result<Statement, ParserError>> {`
Original file line number	Diff line number	Diff line change
`@@ -109,4 +109,8 @@ impl Dialect for RedshiftSqlDialect {`
`109`	`109`	`fn supports_partiql(&self) -> bool {`
`110`	`110`	`true`
`111`	`111`	`}`
	`112`	`+`
	`113`	`+ fn supports_string_escape_constant(&self) -> bool {`
	`114`	`+ true`
	`115`	`+ }`
`112`	`116`	`}`