fix raw query escaping

Author: StpMax

mindsdb_sql_parser/lexer.py

@@ -346,11 +346,13 @@ def INTEGER(self, t):
 
     @_(r"'(?:\\.|[^'])*(?:''(?:\\.|[^'])*)*'")
     def QUOTE_STRING(self, t):
+        t.raw_value = t.value
         t.value = t.value.replace('\\"', '"').replace("\\'", "'").replace("''", "'")
         return t
 
     @_(r'"(?:\\.|[^"])*"')
     def DQUOTE_STRING(self, t):
+        t.raw_value = t.value
         t.value = t.value.replace('\\"', '"').replace("\\'", "'")
         return t
 

mindsdb_sql_parser/parser.py

@@ -1308,7 +1308,7 @@ def from_table_aliased(self, p):
     def from_table(self, p):
         query = NativeQuery(
             integration=p.identifier,
-            query=tokens_to_string(p.raw_query)
+            query=tokens_to_string(p.raw_query, use_raw_values=True)
         )
         return query
 

mindsdb_sql_parser/utils.py

@@ -59,7 +59,7 @@ def to_single_line(text):
     return text
 
 
-def tokens_to_string(tokens):
+def tokens_to_string(tokens, use_raw_values: bool = False):
     # converts list of token (after lexer) to original string
 
     line_num = tokens[0].lineno
@@ -81,10 +81,15 @@ def tokens_to_string(tokens):
         # filling space between tokens
         line += ' '*(token.index - shift - len(line))
 
+        if use_raw_values and hasattr(token, 'raw_value'):
+            value = token.raw_value or token.value
+        else:
+            value = token.value
+
         # add token
-        line += token.value
+        line += value
 
-        last_pos = token.index + len(token.value)
+        last_pos = token.index + len(value)
 
     # last line
     content += line

sly/lex.py

@@ -74,9 +74,9 @@ class Token(object):
     '''
     Representation of a single token.
     '''
-    __slots__ = ('type', 'value', 'lineno', 'index', 'end')
+    __slots__ = ('type', 'value', 'lineno', 'index', 'end', 'raw_value')
     def __repr__(self):
-        return f'Token(type={self.type!r}, value={self.value!r}, lineno={self.lineno}, index={self.index}, end={self.end})'
+        return f'Token(type={self.type!r}, value={self.value!r}, lineno={self.lineno}, index={self.index}, end={self.end}, raw_value={self.raw_value!r})'
 
 class TokenStr(str):
     @staticmethod