Skip to content

Commit eeb5613

Browse files
ea-rusea-rus
authored
Merge pull request #69 from mindsdb/fix-json-render
Update standard render: use own dicts serializer instead of json.dump
2 parents 5885d02 + 286f009 commit eeb5613

File tree

7 files changed

+94
-17
lines changed

7 files changed

+94
-17
lines changed

‎mindsdb_sql_parser/ast/mindsdb/create_database.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
import json
21
from mindsdb_sql_parser.ast.base import ASTNode
3-
from mindsdb_sql_parser.utils import indent
2+
from mindsdb_sql_parser.utils import indent, dump_json
43

54

65
class CreateDatabase(ASTNode):
@@ -49,6 +48,6 @@ def get_string(self, *args, **kwargs):
4948

5049
parameters_str = ''
5150
if self.parameters:
52-
parameters_str = f', PARAMETERS = {json.dumps(self.parameters)}'
51+
parameters_str = f', PARAMETERS = {dump_json(self.parameters)}'
5352
out_str = f'CREATE{replace_str} DATABASE {"IF NOT EXISTS " if self.if_not_exists else ""}{self.name.to_string()} {engine_str}{parameters_str}'
5453
return out_str

‎mindsdb_sql_parser/ast/mindsdb/create_predictor.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
import json
21
from mindsdb_sql_parser.ast.base import ASTNode
3-
from mindsdb_sql_parser.utils import indent
2+
from mindsdb_sql_parser.utils import indent, dump_json
43
from mindsdb_sql_parser.ast.select import Identifier
54
from mindsdb_sql_parser.ast.select.operation import Object
65

@@ -101,13 +100,13 @@ def get_string(self, *args, **kwargs):
101100
for key, value in self.using.items():
102101
if isinstance(value, Object):
103102
args = [
104-
f'{k}={json.dumps(v)}'
103+
f'{k}={dump_json(v)}'
105104
for k, v in value.params.items()
106105
]
107106
args_str = ', '.join(args)
108107
value = f'{value.type}({args_str})'
109108
else:
110-
value = json.dumps(value)
109+
value = dump_json(value)
111110

112111
using_ar.append(f'{Identifier(key).to_string()}={value}')
113112

‎mindsdb_sql_parser/ast/select/select.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
from typing import List, Union
2-
import json
32
from mindsdb_sql_parser.ast.base import ASTNode
4-
from mindsdb_sql_parser.utils import indent
3+
from mindsdb_sql_parser.utils import indent, dump_json
54
from mindsdb_sql_parser.ast.select.operation import Object
65

6+
77
class Select(ASTNode):
88

99
def __init__(self,
@@ -158,15 +158,15 @@ def get_string(self, *args, **kwargs):
158158
for key, value in self.using.items():
159159
if isinstance(value, Object):
160160
args = [
161-
f'{k}={json.dumps(v)}'
161+
f'{k}={dump_json(v)}'
162162
for k, v in value.params.items()
163163
]
164164
args_str = ', '.join(args)
165165
value = f'{value.type}({args_str})'
166166
if isinstance(value, Identifier):
167167
value = value.to_string()
168168
else:
169-
value = json.dumps(value)
169+
value = dump_json(value)
170170

171171
using_ar.append(f'{Identifier(key).to_string()}={value}')
172172

‎mindsdb_sql_parser/lexer.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -353,7 +353,7 @@ def INTEGER(self, t):
353353
def QUOTE_STRING(self, t):
354354
return t
355355

356-
@_(r'"(?:\\.|[^"])*"')
356+
@_(r'"(?:\\.|[^"])*(?:""(?:\\.|[^"])*)*"')
357357
def DQUOTE_STRING(self, t):
358358
return t
359359

‎mindsdb_sql_parser/parser.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from mindsdb_sql_parser.exceptions import ParsingException
2727
from mindsdb_sql_parser.ast.mindsdb.retrain_predictor import RetrainPredictor
2828
from mindsdb_sql_parser.ast.mindsdb.finetune_predictor import FinetunePredictor
29-
from mindsdb_sql_parser.utils import ensure_select_keyword_order, JoinType, tokens_to_string
29+
from mindsdb_sql_parser.utils import ensure_select_keyword_order, JoinType, tokens_to_string, unquote
3030
from mindsdb_sql_parser.logger import ParserLogger
3131

3232
from mindsdb_sql_parser.lexer import MindsDBLexer
@@ -2024,11 +2024,11 @@ def integer(self, p):
20242024

20252025
@_('QUOTE_STRING')
20262026
def quote_string(self, p):
2027-
return p[0].replace('\\"', '"').replace("\\'", "'").replace("''", "'").strip('\'')
2027+
return unquote(p[0]).strip('\'')
20282028

20292029
@_('DQUOTE_STRING')
20302030
def dquote_string(self, p):
2031-
return p[0].replace('\\"', '"').replace("\\'", "'").strip('\"')
2031+
return unquote(p[0], is_double_quoted=True).strip('\"')
20322032

20332033
# for raw query
20342034

‎mindsdb_sql_parser/utils.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,3 +89,57 @@ def tokens_to_string(tokens):
8989
# last line
9090
content += line
9191
return content
92+
93+
94+
def unquote(s, is_double_quoted=False):
95+
s = s.replace('\\"', '"').replace("\\'", "'")
96+
if is_double_quoted:
97+
s = s.replace('""', '"')
98+
else:
99+
s = s.replace("''", "'")
100+
return s
101+
102+
103+
def dump_json(obj) -> str:
104+
'''
105+
dump dict into json-like string using:
106+
- single quotes for strings
107+
- the same quoting rules as `unquote` function
108+
'''
109+
110+
111+
if isinstance(obj, dict):
112+
items = []
113+
for k, v in obj.items():
114+
# keys must be strings in JSON
115+
if not isinstance(k, str):
116+
k = str(k)
117+
items.append(f'{dump_json(k)}: {dump_json(v)}')
118+
return "{" + ", ".join(items) + "}"
119+
120+
if isinstance(obj, (list, tuple)):
121+
items = [
122+
dump_json(i) for i in obj
123+
]
124+
return "[" + ", ".join(items) + "]"
125+
126+
if isinstance(obj, str):
127+
obj = obj.replace("'", "''")
128+
return f"'{obj}'"
129+
130+
if isinstance(obj, (int, float)):
131+
if obj != obj: # NaN
132+
return "null"
133+
if obj == float('inf'):
134+
return "null"
135+
if obj == float('-inf'):
136+
return "null"
137+
return str(obj)
138+
139+
if obj is None:
140+
return "null"
141+
142+
if isinstance(obj, bool):
143+
return "true" if obj else "false"
144+
145+
return dump_json(str(obj))

‎tests/test_mindsdb/test_databases.py‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,6 @@ def test_create_project(self):
119119
assert str(ast).lower() == str(expected_ast).lower()
120120
assert ast.to_tree() == expected_ast.to_tree()
121121

122-
123122
def test_create_database_using(self):
124123

125124
sql = "CREATE DATABASE db using ENGINE = 'mysql', PARAMETERS = {'A': 1}"
@@ -130,7 +129,6 @@ def test_create_database_using(self):
130129
assert str(ast).lower() == str(expected_ast).lower()
131130
assert ast.to_tree() == expected_ast.to_tree()
132131

133-
134132
def test_alter_database(self):
135133
sql = "ALTER DATABASE db PARAMETERS = {'A': 1, 'B': 2}"
136134
ast = parse_sql(sql)
@@ -139,3 +137,30 @@ def test_alter_database(self):
139137

140138
assert str(ast) == str(expected_ast)
141139
assert ast.to_tree() == expected_ast.to_tree()
140+
141+
def test_parser_render(self):
142+
143+
value = "a dm\\in123\"_.,';:!@#$%^&*()\n<>`{}[]"
144+
145+
'''
146+
quoting rules:
147+
' => '' (in single quoted strings)
148+
" => "" (in double quoted strings)
149+
'''
150+
for symbol in ("'", '"'):
151+
sql = f"""
152+
CREATE DATABASE db WITH engine = 'postgres'
153+
PARAMETERS = {{
154+
'password': {symbol}{value.replace(symbol, symbol * 2)}{symbol}
155+
}}
156+
"""
157+
158+
# check parsing
159+
query = parse_sql(sql)
160+
assert query.parameters['password'] == value
161+
162+
# check render
163+
sql2 = str(query)
164+
query2 = parse_sql(sql2)
165+
assert query2.parameters['password'] == value
166+

0 commit comments

Comments
 (0)