fix!: add rust config for noop string escapes

Author: nian0114

sqlglot/tokens.py

@@ -600,6 +600,7 @@ def _quotes_to_format(
                 tokens_preceding_hint={
                     _TOKEN_TYPE_TO_INDEX[v] for v in klass.TOKENS_PRECEDING_HINT
                 },
+                string_escapes_noop=klass._STRING_ESCAPES_NOOP,
             )
             token_types = RsTokenTypeSettings(
                 bit_string=_TOKEN_TYPE_TO_INDEX[TokenType.BIT_STRING],

sqlglotrs/benches/tokenizer_settings.json

@@ -1 +1 @@
-{"white_space":{"\n":55,"\t":54,"\r":55," ":54},"single_tokens":{"\"":320,",":6,".":7,"[":2,"*":14,":":10,"]":3,"'":320,"(":0,")":1,"?":311,"-":8,"@":47,"$":46},"keywords":{"..":7},"numeric_literals":{},"identifiers":{"\"":"\""},"identifier_escapes":["\\"],"string_escapes":["\\"],"quotes":{"'":"'"},"format_strings":{"N'":["'",70],"n'":["'",70]},"has_bit_strings":false,"has_hex_strings":false,"comments":{"{#":"#}","--":null,"/*":"*/"},"var_single_tokens":[],"commands":[237,341,205,234,324],"command_prefix_tokens":[13,197],"tokens_preceding_hint":[261,334,221,361],"heredoc_tag_is_identifier":false,"string_escapes_allowed_in_raw_strings":true,"nested_comments":true,"hint_start":"/*+"}
+{"white_space":{"\n":55,"\t":54,"\r":55," ":54},"single_tokens":{"\"":320,",":6,".":7,"[":2,"*":14,":":10,"]":3,"'":320,"(":0,")":1,"?":311,"-":8,"@":47,"$":46},"keywords":{"..":7},"numeric_literals":{},"identifiers":{"\"":"\""},"identifier_escapes":["\\"],"string_escapes":["\\"],"quotes":{"'":"'"},"format_strings":{"N'":["'",70],"n'":["'",70]},"has_bit_strings":false,"has_hex_strings":false,"comments":{"{#":"#}","--":null,"/*":"*/"},"var_single_tokens":[],"commands":[237,341,205,234,324],"command_prefix_tokens":[13,197],"tokens_preceding_hint":[261,334,221,361],"heredoc_tag_is_identifier":false,"string_escapes_allowed_in_raw_strings":true,"nested_comments":true,"hint_start":"/*+","string_escapes_noop":[]}

sqlglotrs/src/settings.rs

@@ -104,6 +104,7 @@ pub struct TokenizerSettings {
     pub string_escapes_allowed_in_raw_strings: bool,
     pub nested_comments: bool,
     pub hint_start: String,
+    pub string_escapes_noop: HashSet<char>,
 }
 
 #[pymethods]
@@ -130,6 +131,7 @@ impl TokenizerSettings {
         string_escapes_allowed_in_raw_strings: bool,
         nested_comments: bool,
         hint_start: String,
+        string_escapes_noop: HashSet<String>,
     ) -> Self {
         let to_char = |v: &String| {
             if v.len() == 1 {
@@ -158,6 +160,7 @@ impl TokenizerSettings {
             identifier_escapes.iter().map(&to_char).collect();
 
         let string_escapes_native: HashSet<char> = string_escapes.iter().map(&to_char).collect();
+        let string_escapes_noop_native: HashSet<char> = string_escapes_noop.iter().map(&to_char).collect();
 
         let var_single_tokens_native: HashSet<char> =
             var_single_tokens.iter().map(&to_char).collect();
@@ -183,6 +186,7 @@ impl TokenizerSettings {
             string_escapes_allowed_in_raw_strings,
             nested_comments,
             hint_start,
+            string_escapes_noop: string_escapes_noop_native,
         };
 
         #[cfg(feature = "profiling")]

sqlglotrs/src/tokenizer.rs

@@ -665,9 +665,17 @@ impl<'a> TokenizerState<'a> {
             {
                 let peek_char_str = self.peek_char.to_string();
                 let equal_delimiter = delimiter == peek_char_str;
-                if equal_delimiter || escapes.contains(&self.peek_char) {
+                
+                if equal_delimiter 
+                    || escapes.contains(&self.peek_char) 
+                    || self.settings.string_escapes_noop.contains(&self.peek_char)
+                {
                     if equal_delimiter {
                         text.push(self.peek_char);
+                    } else if self.settings.string_escapes_noop.contains(&self.peek_char)
+                        && self.current_char != self.peek_char 
+                    {
+                        text.push(self.peek_char);
                     } else {
                         text.push(self.current_char);
                         text.push(self.peek_char);