feat!: add support for noop string escapes

This commit let dialects (like MySQL) allow escaping quotes that are not the current string delimiter, effectively treating the escape character as a no-op regarding the character's literal value but consuming the backslash. Previously, tokenize("""'\"a'""", "mysql") resulted in "a. With this change, it correctly results in "a.
Example:
In MySQL:
'"' -> " (Backslash is consumed)
'\"' -> " (Backslash is escaped, quote remains)

Author: nian0114

sqlglot/dialects/mysql.py

@@ -202,6 +202,7 @@ class Tokenizer(tokens.Tokenizer):
         STRING_ESCAPES = ["'", '"', "\\"]
         BIT_STRINGS = [("b'", "'"), ("B'", "'"), ("0b", "")]
         HEX_STRINGS = [("x'", "'"), ("X'", "'"), ("0x", "")]
+        STRING_ESCAPES_NOOP = ['"', "'"]
 
         NESTED_COMMENTS = False
 

sqlglot/tokens.py

@@ -548,6 +548,7 @@ def _quotes_to_format(
         }
 
         klass._STRING_ESCAPES = set(klass.STRING_ESCAPES)
+        klass._STRING_ESCAPES_NOOP = set(klass.STRING_ESCAPES_NOOP)
         klass._IDENTIFIER_ESCAPES = set(klass.IDENTIFIER_ESCAPES)
         klass._COMMENTS = {
             **dict(
@@ -668,6 +669,7 @@ class Tokenizer(metaclass=_Tokenizer):
     QUOTES: t.List[t.Tuple[str, str] | str] = ["'"]
     STRING_ESCAPES = ["'"]
     VAR_SINGLE_TOKENS: t.Set[str] = set()
+    STRING_ESCAPES_NOOP: t.List[str] = []
 
     # The strings in this list can always be used as escapes, regardless of the surrounding
     # identifier delimiters. By default, the closing delimiter is assumed to also act as an
@@ -698,6 +700,7 @@ class Tokenizer(metaclass=_Tokenizer):
     _STRING_ESCAPES: t.Set[str] = set()
     _KEYWORD_TRIE: t.Dict = {}
     _RS_TOKENIZER: t.Optional[t.Any] = None
+    _STRING_ESCAPES_NOOP: t.Set[str] = set()
 
     KEYWORDS: t.Dict[str, TokenType] = {
         **{f"{{%{postfix}": TokenType.BLOCK_START for postfix in ("", "+", "-")},
@@ -1509,14 +1512,21 @@ def _extract_string(
                     self._advance(2)
                     text += unescaped_sequence
                     continue
+
             if (
                 (self.STRING_ESCAPES_ALLOWED_IN_RAW_STRINGS or not raw_string)
                 and self._char in escapes
-                and (self._peek == delimiter or self._peek in escapes)
+                and (
+                    self._peek == delimiter
+                    or self._peek in escapes
+                    or self._peek in self.STRING_ESCAPES_NOOP
+                )
                 and (self._char not in self._QUOTES or self._char == self._peek)
             ):
                 if self._peek == delimiter:
                     text += self._peek
+                elif self._peek in self.STRING_ESCAPES_NOOP and self._char != self._peek:
+                    text += self._peek
                 else:
                     text += self._char + self._peek
 

tests/dialects/test_mysql.py

@@ -432,6 +432,19 @@ def test_escape(self):
             },
         )
 
+        self.validate_all(
+            r"'\"'",
+            write={
+                "mysql": """\'"\'""",
+            },
+        )
+        self.validate_all(
+            "'\\\\\"a'",
+            write={
+                "mysql": "'\\\\\"a'",
+            },
+        )
+
     def test_introducers(self):
         self.validate_all(
             "_utf8mb4 'hola'",