Use all_dialects_where to test backslash escape for every dialect

Author: hansott

src/test_utils.rs

@@ -33,7 +33,7 @@ use core::fmt::Debug;
 
 use crate::dialect::*;
 use crate::parser::{Parser, ParserError};
-use crate::tokenizer::Tokenizer;
+use crate::tokenizer::{Tokenizer, Token};
 use crate::{ast::*, parser::ParserOptions};
 
 #[cfg(test)]
@@ -237,6 +237,18 @@ impl TestedDialects {
     pub fn verified_expr(&self, sql: &str) -> Expr {
         self.expr_parses_to(sql, sql)
     }
+
+    /// Check that the tokenizer returns the expected tokens for the given SQL.
+    pub fn tokenizes_to(&self, sql: &str, expected: Vec<Token>) {
+        self.dialects.iter().for_each(|dialect| {
+            let mut tokenizer = Tokenizer::new(&**dialect, sql);
+            if let Some(options) = &self.options {
+                tokenizer = tokenizer.with_unescape(options.unescape);
+            }
+            let tokens = tokenizer.tokenize().unwrap();
+            assert_eq!(expected, tokens);
+        });
+    }
 }
 
 /// Returns all available dialects.

src/tokenizer.rs

@@ -2159,6 +2159,7 @@ mod tests {
         BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect,
     };
     use core::fmt::Debug;
+    use crate::test_utils::all_dialects_where;
 
     #[test]
     fn tokenizer_error_impl() {
@@ -3548,15 +3549,30 @@ mod tests {
     }
 
     #[test]
-    fn test_national_strings() {
-        let dialect = PostgreSqlDialect {};
-        let sql = "select n'''''\\'";
-        let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
-        let expected = vec![
-            Token::make_keyword("select"),
-            Token::Whitespace(Whitespace::Space),
-            Token::NationalStringLiteral("''\\".to_string()),
-        ];
-        compare(expected, tokens);
+    fn test_national_strings_backslash_escape_not_supported() {
+        all_dialects_where(
+            |dialect| !dialect.supports_string_literal_backslash_escape(),
+        ).tokenizes_to(
+            "select n'''''\\'",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::NationalStringLiteral("''\\".to_string()),
+            ],
+        );
+    }
+
+    #[test]
+    fn test_national_strings_backslash_escape_supported() {
+        all_dialects_where(
+            |dialect| dialect.supports_string_literal_backslash_escape(),
+        ).tokenizes_to(
+            "select n'''''\\''",
+            vec![
+                Token::make_keyword("select"),
+                Token::Whitespace(Whitespace::Space),
+                Token::NationalStringLiteral("'''".to_string()),
+            ],
+        );
     }
 }