fix!: invert STRING_ESCAPES_NOOP to blacklist mode

Author: nian0114

sqlglot/dialects/mysql.py

@@ -202,7 +202,8 @@ class Tokenizer(tokens.Tokenizer):
         STRING_ESCAPES = ["'", '"', "\\"]
         BIT_STRINGS = [("b'", "'"), ("B'", "'"), ("0b", "")]
         HEX_STRINGS = [("x'", "'"), ("X'", "'"), ("0x", "")]
-        STRING_ESCAPES_NOOP = ['"', "'"]
+        # https://dev.mysql.com/doc/refman/8.4/en/string-literals.html
+        ESCAPE_FOLLOW_CHARS = ["0", "b", "n", "r", "t", "Z", "%", "_"]
 
         NESTED_COMMENTS = False
 

sqlglot/tokens.py

@@ -548,7 +548,7 @@ def _quotes_to_format(
         }
 
         klass._STRING_ESCAPES = set(klass.STRING_ESCAPES)
-        klass._STRING_ESCAPES_NOOP = set(klass.STRING_ESCAPES_NOOP)
+        klass._ESCAPE_FOLLOW_CHARS = set(klass.ESCAPE_FOLLOW_CHARS)
         klass._IDENTIFIER_ESCAPES = set(klass.IDENTIFIER_ESCAPES)
         klass._COMMENTS = {
             **dict(
@@ -600,7 +600,7 @@ def _quotes_to_format(
                 tokens_preceding_hint={
                     _TOKEN_TYPE_TO_INDEX[v] for v in klass.TOKENS_PRECEDING_HINT
                 },
-                string_escapes_noop=klass._STRING_ESCAPES_NOOP,
+                escape_follow_chars=klass._ESCAPE_FOLLOW_CHARS,
             )
             token_types = RsTokenTypeSettings(
                 bit_string=_TOKEN_TYPE_TO_INDEX[TokenType.BIT_STRING],
@@ -670,7 +670,7 @@ class Tokenizer(metaclass=_Tokenizer):
     QUOTES: t.List[t.Tuple[str, str] | str] = ["'"]
     STRING_ESCAPES = ["'"]
     VAR_SINGLE_TOKENS: t.Set[str] = set()
-    STRING_ESCAPES_NOOP: t.List[str] = []
+    ESCAPE_FOLLOW_CHARS: t.List[str] = []
 
     # The strings in this list can always be used as escapes, regardless of the surrounding
     # identifier delimiters. By default, the closing delimiter is assumed to also act as an
@@ -701,7 +701,7 @@ class Tokenizer(metaclass=_Tokenizer):
     _STRING_ESCAPES: t.Set[str] = set()
     _KEYWORD_TRIE: t.Dict = {}
     _RS_TOKENIZER: t.Optional[t.Any] = None
-    _STRING_ESCAPES_NOOP: t.Set[str] = set()
+    _ESCAPE_FOLLOW_CHARS: t.Set[str] = set()
 
     KEYWORDS: t.Dict[str, TokenType] = {
         **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")},
@@ -1514,19 +1514,21 @@ def _extract_string(
                     text += unescaped_sequence
                     continue
 
+            is_valid_custom_escape = (
+                self.ESCAPE_FOLLOW_CHARS
+                and self._char == "\\"
+                and self._peek not in self.ESCAPE_FOLLOW_CHARS
+            )
+
             if (
                 (self.STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS or not raw_string)
                 and self._char in escapes
-                and (
-                    self._peek == delimiter
-                    or self._peek in escapes
-                    or self._peek in self.STRING_ESCAPES_NOOP
-                )
+                and (self._peek == delimiter or self._peek in escapes or is_valid_custom_escape)
                 and (self._char not in self._QUOTES or self._char == self._peek)
             ):
                 if self._peek == delimiter:
                     text += self._peek
-                elif self._peek in self.STRING_ESCAPES_NOOP and self._char != self._peek:
+                elif is_valid_custom_escape and self._char != self._peek:
                     text += self._peek
                 else:
                     text += self._char + self._peek

sqlglotrs/benches/tokenizer_settings.json

@@ -1 +1 @@
-{"white_space":{"\n":55,"\t":54,"\r":55," ":54},"single_tokens":{"\"":320,",":6,".":7,"[":2,"*":14,":":10,"]":3,"'":320,"(":0,")":1,"?":311,"-":8,"@":47,"$":46},"keywords":{"..":7},"numeric_literals":{},"identifiers":{"\"":"\""},"identifier_escapes":["\\"],"string_escapes":["\\"],"quotes":{"'":"'"},"format_strings":{"N'":["'",70],"n'":["'",70]},"has_bit_strings":false,"has_hex_strings":false,"comments":{"{#":"#}","--":null,"/*":"*/"},"var_single_tokens":[],"commands":[237,341,205,234,324],"command_prefix_tokens":[13,197],"tokens_preceding_hint":[261,334,221,361],"heredoc_tag_is_identifier":false,"string_escapes_allowed_in_raw_strings":true,"nested_comments":true,"hint_start":"/*+","string_escapes_noop":[]}
+{"white_space":{"\n":55,"\t":54,"\r":55," ":54},"single_tokens":{"\"":320,",":6,".":7,"[":2,"*":14,":":10,"]":3,"'":320,"(":0,")":1,"?":311,"-":8,"@":47,"$":46},"keywords":{"..":7},"numeric_literals":{},"identifiers":{"\"":"\""},"identifier_escapes":["\\"],"string_escapes":["\\"],"quotes":{"'":"'"},"format_strings":{"N'":["'",70],"n'":["'",70]},"has_bit_strings":false,"has_hex_strings":false,"comments":{"{#":"#}","--":null,"/*":"*/"},"var_single_tokens":[],"commands":[237,341,205,234,324],"command_prefix_tokens":[13,197],"tokens_preceding_hint":[261,334,221,361],"heredoc_tag_is_identifier":false,"string_escapes_allowed_in_raw_strings":true,"nested_comments":true,"hint_start":"/*+","escape_follow_chars":[]}

sqlglotrs/src/settings.rs

@@ -104,7 +104,7 @@ pub struct TokenizerSettings {
     pub string_escapes_allowed_in_raw_strings: bool,
     pub nested_comments: bool,
     pub hint_start: String,
-    pub string_escapes_noop: HashSet<char>,
+    pub escape_follow_chars: HashSet<char>,
 }
 
 #[pymethods]
@@ -131,7 +131,7 @@ impl TokenizerSettings {
         string_escapes_allowed_in_raw_strings: bool,
         nested_comments: bool,
         hint_start: String,
-        string_escapes_noop: HashSet<String>,
+        escape_follow_chars: HashSet<String>,
     ) -> Self {
         let to_char = |v: &String| {
             if v.len() == 1 {
@@ -160,7 +160,7 @@ impl TokenizerSettings {
             identifier_escapes.iter().map(&to_char).collect();
 
         let string_escapes_native: HashSet<char> = string_escapes.iter().map(&to_char).collect();
-        let string_escapes_noop_native: HashSet<char> = string_escapes_noop.iter().map(&to_char).collect();
+        let escape_follow_chars_native: HashSet<char> = escape_follow_chars.iter().map(&to_char).collect();
 
         let var_single_tokens_native: HashSet<char> =
             var_single_tokens.iter().map(&to_char).collect();
@@ -186,7 +186,7 @@ impl TokenizerSettings {
             string_escapes_allowed_in_raw_strings,
             nested_comments,
             hint_start,
-            string_escapes_noop: string_escapes_noop_native,
+            escape_follow_chars: escape_follow_chars_native,
         };
 
         #[cfg(feature = "profiling")]

sqlglotrs/src/tokenizer.rs

@@ -665,14 +665,18 @@ impl<'a> TokenizerState<'a> {
             {
                 let peek_char_str = self.peek_char.to_string();
                 let equal_delimiter = delimiter == peek_char_str;
-                
+                let is_valid_custom_escape =
+                    self.current_char == '\\'
+                        && !self.settings.escape_follow_chars.is_empty()
+                        && !self.settings.escape_follow_chars.contains(&self.peek_char);
+
                 if equal_delimiter 
                     || escapes.contains(&self.peek_char) 
-                    || self.settings.string_escapes_noop.contains(&self.peek_char)
+                    || is_valid_custom_escape
                 {
                     if equal_delimiter {
                         text.push(self.peek_char);
-                    } else if self.settings.string_escapes_noop.contains(&self.peek_char)
+                    } else if is_valid_custom_escape
                         && self.current_char != self.peek_char 
                     {
                         text.push(self.peek_char);

tests/dialects/test_mysql.py

@@ -432,17 +432,21 @@ def test_escape(self):
             },
         )
 
-        self.validate_all(
+        self.validate_identity(
             r"'\"'",
-            write={
-                "mysql": """\'"\'""",
-            },
+            """\'"\'""",
         )
-        self.validate_all(
+        self.validate_identity(
             "'\\\\\"a'",
-            write={
-                "mysql": "'\\\\\"a'",
-            },
+            "'\\\\\"a'",
+        )
+        self.validate_identity(
+            "'\t'",
+            "'\\t'",
+        )
+        self.validate_identity(
+            "'\j'",
+            "'j'",
         )
 
     def test_introducers(self):