Skip to content

Commit bd8ba44

Browse files
lukasstadlerlukasstadler
committed
implement _json module
1 parent f5ef364 commit bd8ba44

File tree

12 files changed

+1356
-17
lines changed

12 files changed

+1356
-17
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,9 @@
124124
import com.oracle.graal.python.builtins.modules.io.StringIOBuiltins;
125125
import com.oracle.graal.python.builtins.modules.io.TextIOBaseBuiltins;
126126
import com.oracle.graal.python.builtins.modules.io.TextIOWrapperBuiltins;
127+
import com.oracle.graal.python.builtins.modules.json.JSONEncoderBuiltins;
128+
import com.oracle.graal.python.builtins.modules.json.JSONModuleBuiltins;
129+
import com.oracle.graal.python.builtins.modules.json.JSONScannerBuiltins;
127130
import com.oracle.graal.python.builtins.modules.lzma.LZMACompressorBuiltins;
128131
import com.oracle.graal.python.builtins.modules.lzma.LZMADecompressorBuiltins;
129132
import com.oracle.graal.python.builtins.modules.lzma.LZMAModuleBuiltins;
@@ -438,6 +441,7 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
438441
new CollectionsModuleBuiltins(),
439442
new JavaModuleBuiltins(),
440443
new JArrayModuleBuiltins(),
444+
new JSONModuleBuiltins(),
441445
new SREModuleBuiltins(),
442446
new AstModuleBuiltins(),
443447
new SelectModuleBuiltins(),
@@ -496,7 +500,11 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
496500
new MultiprocessingModuleBuiltins(),
497501
new SemLockBuiltins(),
498502
new WarningsModuleBuiltins(),
499-
new GraalPythonModuleBuiltins()));
503+
new GraalPythonModuleBuiltins(),
504+
505+
// json
506+
new JSONScannerBuiltins(),
507+
new JSONEncoderBuiltins()));
500508
if (hasCoverageTool) {
501509
builtins.add(new TraceModuleBuiltins());
502510
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,10 @@ public enum PythonBuiltinClassType implements TruffleObject {
202202
PSSLSocket("_SSLSocket", "_ssl"),
203203
PMemoryBIO("MemoryBIO", "_ssl"),
204204

205+
// json
206+
JSONScanner("Scanner", "_json", Flags.PUBLIC_BASE_WODICT),
207+
JSONEncoder("Encoder", "_json", Flags.PUBLIC_BASE_WODICT),
208+
205209
// Errors and exceptions:
206210

207211
// everything after BaseException is considered to be an exception

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1158,7 +1158,7 @@ private static Object stringToIntInternal(String num, int base) {
11581158

11591159
private Object stringToInt(VirtualFrame frame, Object cls, String number, int base, Object origObj) {
11601160
if (base == 0 || base == 10) {
1161-
Object value = parseSimpleDecimalLiteral(number);
1161+
Object value = parseSimpleDecimalLiteral(number, 0, number.length());
11621162
if (value != null) {
11631163
return createInt(cls, value);
11641164
}
@@ -1325,23 +1325,23 @@ private static BigInteger asciiToBigInteger(String str, int possibleBase) throws
13251325
* @param arg the string to parse
13261326
* @return parsed integer, long or null if the literal is not simple enough
13271327
*/
1328-
private static Object parseSimpleDecimalLiteral(String arg) {
1329-
if (arg.isEmpty()) {
1328+
public static Object parseSimpleDecimalLiteral(String arg, int offset, int remaining) {
1329+
if (remaining <= 0) {
13301330
return null;
13311331
}
1332-
int start = arg.charAt(0) == '-' ? 1 : 0;
1333-
if (arg.length() <= start || arg.length() > 18 + start) {
1332+
int start = arg.charAt(offset) == '-' ? 1 : 0;
1333+
if (remaining <= start || remaining > 18 + start) {
13341334
return null;
13351335
}
1336-
if (arg.charAt(start) == '0') {
1337-
if (arg.length() > start + 1) {
1336+
if (arg.charAt(start + offset) == '0') {
1337+
if (remaining > start + 1) {
13381338
return null;
13391339
}
13401340
return 0;
13411341
}
13421342
long value = 0;
1343-
for (int i = start; i < arg.length(); i++) {
1344-
char c = arg.charAt(i);
1343+
for (int i = start; i < remaining; i++) {
1344+
char c = arg.charAt(i + offset);
13451345
if (c < '0' || c > '9') {
13461346
return null;
13471347
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/json/JSONEncoderBuiltins.java

Lines changed: 302 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,302 @@
1+
/* Copyright (c) 2020, 2021, Oracle and/or its affiliates.
2+
* Copyright (C) 1996-2020 Python Software Foundation
3+
*
4+
* Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
5+
*/
6+
package com.oracle.graal.python.builtins.modules.json;
7+
8+
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.TypeError;
9+
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.ValueError;
10+
import static com.oracle.graal.python.nodes.SpecialMethodNames.__CALL__;
11+
12+
import java.util.List;
13+
14+
import com.oracle.graal.python.annotations.ArgumentClinic;
15+
import com.oracle.graal.python.builtins.Builtin;
16+
import com.oracle.graal.python.builtins.CoreFunctions;
17+
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
18+
import com.oracle.graal.python.builtins.PythonBuiltins;
19+
import com.oracle.graal.python.builtins.objects.PNone;
20+
import com.oracle.graal.python.builtins.objects.common.HashingStorage;
21+
import com.oracle.graal.python.builtins.objects.common.HashingStorage.DictEntry;
22+
import com.oracle.graal.python.builtins.objects.common.HashingStorageLibrary;
23+
import com.oracle.graal.python.builtins.objects.common.HashingStorageLibrary.HashingStorageIterable;
24+
import com.oracle.graal.python.builtins.objects.dict.PDict;
25+
import com.oracle.graal.python.builtins.objects.floats.FloatBuiltins;
26+
import com.oracle.graal.python.builtins.objects.floats.PFloat;
27+
import com.oracle.graal.python.builtins.objects.function.PKeyword;
28+
import com.oracle.graal.python.builtins.objects.ints.PInt;
29+
import com.oracle.graal.python.builtins.objects.list.ListBuiltins.ListSortNode;
30+
import com.oracle.graal.python.builtins.objects.list.PList;
31+
import com.oracle.graal.python.builtins.objects.str.PString;
32+
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
33+
import com.oracle.graal.python.nodes.PGuards;
34+
import com.oracle.graal.python.nodes.PRaiseNode;
35+
import com.oracle.graal.python.nodes.SpecialMethodNames;
36+
import com.oracle.graal.python.nodes.builtins.ListNodes.ConstructListNode;
37+
import com.oracle.graal.python.nodes.call.special.CallUnaryMethodNode;
38+
import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
39+
import com.oracle.graal.python.nodes.control.GetNextNode;
40+
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
41+
import com.oracle.graal.python.nodes.function.builtins.PythonTernaryClinicBuiltinNode;
42+
import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider;
43+
import com.oracle.graal.python.nodes.object.GetClassNode;
44+
import com.oracle.graal.python.nodes.object.IsBuiltinClassProfile;
45+
import com.oracle.graal.python.nodes.util.CastToJavaStringNode;
46+
import com.oracle.graal.python.runtime.exception.PException;
47+
import com.oracle.graal.python.runtime.formatting.FloatFormatter;
48+
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
49+
import com.oracle.graal.python.runtime.sequence.PSequence;
50+
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
51+
import com.oracle.graal.python.util.PythonUtils;
52+
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
53+
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
54+
import com.oracle.truffle.api.dsl.NodeFactory;
55+
import com.oracle.truffle.api.dsl.Specialization;
56+
57+
@CoreFunctions(extendClasses = PythonBuiltinClassType.JSONEncoder)
58+
public class JSONEncoderBuiltins extends PythonBuiltins {
59+
60+
@Override
61+
protected List<? extends NodeFactory<? extends PythonBuiltinBaseNode>> getNodeFactories() {
62+
return JSONEncoderBuiltinsFactory.getFactories();
63+
}
64+
65+
@Builtin(name = __CALL__, minNumOfPositionalArgs = 1, parameterNames = {"$self", "obj", "_current_indent_level"})
66+
@ArgumentClinic(name = "_current_indent_level", conversion = ArgumentClinic.ClinicConversion.Int, defaultValue = "0", useDefaultForNone = true)
67+
@GenerateNodeFactory
68+
public abstract static class CallEncoderNode extends PythonTernaryClinicBuiltinNode {
69+
70+
@Child private CallUnaryMethodNode callEncode = CallUnaryMethodNode.create();
71+
@Child private CallUnaryMethodNode callDefaultFn = CallUnaryMethodNode.create();
72+
@Child private CastToJavaStringNode castEncodeResult = CastToJavaStringNode.create();
73+
@Child private LookupAndCallUnaryNode callGetItems = LookupAndCallUnaryNode.create(SpecialMethodNames.ITEMS);
74+
@Child private LookupAndCallUnaryNode callGetIter = LookupAndCallUnaryNode.create(SpecialMethodNames.__ITER__);
75+
@Child private HashingStorageLibrary dictLib = HashingStorageLibrary.getFactory().createDispatched(6);
76+
@Child private ListSortNode sortList = ListSortNode.create();
77+
@Child private IsBuiltinClassProfile stopDictIterationProfile = IsBuiltinClassProfile.create();
78+
@Child private GetNextNode callNext = GetNextNode.create();
79+
@Child private GetClassNode getDictClass = GetClassNode.create();
80+
@Child private ConstructListNode constructList = ConstructListNode.create();
81+
82+
@Child private PythonObjectFactory factory = PythonObjectFactory.create();
83+
84+
@Override
85+
protected ArgumentClinicProvider getArgumentClinic() {
86+
return JSONEncoderBuiltinsClinicProviders.CallEncoderNodeClinicProviderGen.INSTANCE;
87+
}
88+
89+
@Specialization
90+
@TruffleBoundary
91+
protected PTuple call(PJSONEncoder self, Object obj, int indent) {
92+
StringBuilder builder = new StringBuilder();
93+
appendListObj(self, builder, obj, indent);
94+
return factory.createTuple(new Object[]{builder.toString()});
95+
}
96+
97+
private static void appendConst(StringBuilder builder, Object obj) {
98+
if (obj == PNone.NONE) {
99+
builder.append("null");
100+
} else if (obj == Boolean.TRUE) {
101+
builder.append("true");
102+
} else {
103+
assert obj == Boolean.FALSE;
104+
builder.append("false");
105+
}
106+
}
107+
108+
private void appendFloat(PJSONEncoder encoder, StringBuilder builder, double obj) {
109+
if (!Double.isFinite(obj)) {
110+
if (!encoder.allowNan) {
111+
throw raise(ValueError, "Out of range float values are not JSON compliant");
112+
}
113+
if (obj > 0) {
114+
builder.append("Infinity");
115+
} else if (obj < 0) {
116+
builder.append("-Infinity");
117+
} else {
118+
builder.append("NaN");
119+
}
120+
} else {
121+
FloatFormatter f = new FloatFormatter(PRaiseNode.getUncached(), FloatBuiltins.StrNode.spec);
122+
f.setMinFracDigits(1);
123+
builder.append(FloatBuiltins.StrNode.doFormat(obj, f));
124+
}
125+
}
126+
127+
private void appendString(PJSONEncoder encoder, StringBuilder builder, String obj) {
128+
switch (encoder.fastEncode) {
129+
case FastEncode:
130+
JSONModuleBuiltins.EncodeBaseString.appendString(obj, builder);
131+
break;
132+
case FastEncodeAscii:
133+
JSONModuleBuiltins.EncodeBaseStringAscii.appendString(obj, builder);
134+
break;
135+
case None:
136+
Object result = callEncode.executeObject(encoder.encoder, obj);
137+
if (!PGuards.isString(result)) {
138+
throw raise(TypeError, "encoder() must return a string, not %p", result);
139+
}
140+
builder.append(castEncodeResult.execute(result));
141+
break;
142+
default:
143+
assert false;
144+
break;
145+
}
146+
}
147+
148+
private boolean appendSimpleObj(PJSONEncoder encoder, StringBuilder builder, Object obj) {
149+
if (obj == PNone.NONE || obj == Boolean.TRUE || obj == Boolean.FALSE) {
150+
appendConst(builder, obj);
151+
} else if (obj instanceof String) {
152+
appendString(encoder, builder, (String) obj);
153+
} else if (obj instanceof PString) {
154+
appendString(encoder, builder, ((PString) obj).toString());
155+
} else if (obj instanceof Integer) {
156+
builder.append((int) obj);
157+
} else if (obj instanceof Long) {
158+
builder.append((long) obj);
159+
} else if (obj instanceof PInt) {
160+
builder.append(((PInt) obj).toString());
161+
} else if (obj instanceof Float) {
162+
appendFloat(encoder, builder, (float) obj);
163+
} else if (obj instanceof Double) {
164+
appendFloat(encoder, builder, (double) obj);
165+
} else if (obj instanceof PFloat) {
166+
appendFloat(encoder, builder, ((PFloat) obj).asDouble());
167+
} else {
168+
return false;
169+
}
170+
return true;
171+
}
172+
173+
private void appendListObj(PJSONEncoder encoder, StringBuilder builder, Object obj, int indentLevel) {
174+
if (appendSimpleObj(encoder, builder, obj)) {
175+
// done
176+
} else if (obj instanceof PList || obj instanceof PTuple) {
177+
appendList(encoder, builder, (PSequence) obj, indentLevel);
178+
} else if (obj instanceof PDict) {
179+
appendDict(encoder, builder, (PDict) obj, indentLevel);
180+
} else {
181+
startRecursion(encoder, obj);
182+
Object newObj = callDefaultFn.executeObject(encoder.defaultFn, obj);
183+
appendListObj(encoder, builder, newObj, indentLevel);
184+
endRecursion(encoder, obj);
185+
}
186+
}
187+
188+
private static void endRecursion(PJSONEncoder encoder, Object obj) {
189+
if (encoder.markers != PNone.NONE) {
190+
encoder.circular.remove(obj);
191+
}
192+
}
193+
194+
private void startRecursion(PJSONEncoder encoder, Object obj) {
195+
if (encoder.markers != PNone.NONE) {
196+
if (encoder.circular.containsKey(obj)) {
197+
throw raise(ValueError, "Circular reference detected");
198+
}
199+
encoder.circular.put(obj, null);
200+
}
201+
}
202+
203+
private void appendDict(PJSONEncoder encoder, StringBuilder builder, PDict dict, int indentLevel) {
204+
HashingStorage storage = dict.getDictStorage();
205+
206+
if (dictLib.length(storage) == 0) {
207+
builder.append("{}");
208+
} else {
209+
startRecursion(encoder, dict);
210+
builder.append('{');
211+
212+
if (encoder.indent != PNone.NONE) {
213+
indentLevel++;
214+
}
215+
216+
if (!encoder.sortKeys && IsBuiltinClassProfile.profileClassSlowPath(getDictClass.execute(dict), PythonBuiltinClassType.PDict)) {
217+
HashingStorageIterable<DictEntry> entries = dictLib.entries(storage);
218+
boolean first = true;
219+
for (DictEntry entry : entries) {
220+
first = appendDictEntry(encoder, builder, indentLevel, first, entry.key, entry.value);
221+
}
222+
} else {
223+
Object items = constructList.execute(null, callGetItems.executeObject(null, dict));
224+
if (encoder.sortKeys) {
225+
sortList.execute(null, items, PythonUtils.EMPTY_OBJECT_ARRAY, PKeyword.EMPTY_KEYWORDS);
226+
}
227+
Object iter = callGetIter.executeObject(null, items);
228+
boolean first = true;
229+
while (true) {
230+
Object item;
231+
try {
232+
item = callNext.execute(null, iter);
233+
} catch (PException e) {
234+
e.expectStopIteration(stopDictIterationProfile);
235+
break;
236+
}
237+
if (!(item instanceof PTuple) || ((PTuple) item).getSequenceStorage().length() != 2) {
238+
throw raise(ValueError, "items must return 2-tuples");
239+
}
240+
SequenceStorage sequenceStorage = ((PTuple) item).getSequenceStorage();
241+
Object key = sequenceStorage.getItemNormalized(0);
242+
Object value = sequenceStorage.getItemNormalized(1);
243+
first = appendDictEntry(encoder, builder, indentLevel, first, key, value);
244+
}
245+
}
246+
247+
builder.append('}');
248+
endRecursion(encoder, dict);
249+
}
250+
}
251+
252+
private boolean appendDictEntry(PJSONEncoder encoder, StringBuilder builder, int indentLevel, boolean first, Object key, Object value) {
253+
if (!first) {
254+
builder.append(encoder.itemSeparator);
255+
}
256+
boolean isString = key instanceof String || key instanceof PString;
257+
if (!isString) {
258+
builder.append('"');
259+
}
260+
if (!appendSimpleObj(encoder, builder, key)) {
261+
if (encoder.skipKeys) {
262+
if (!isString) {
263+
builder.setLength(builder.length() - 1);
264+
}
265+
return true;
266+
}
267+
throw raise(TypeError, "keys must be str, int, float, bool or None, not %p", key);
268+
}
269+
if (!isString) {
270+
builder.append('"');
271+
}
272+
builder.append(encoder.keySeparator);
273+
appendListObj(encoder, builder, value, indentLevel);
274+
return false;
275+
}
276+
277+
private void appendList(PJSONEncoder encoder, StringBuilder builder, PSequence list, int indentLevel) {
278+
SequenceStorage storage = list.getSequenceStorage();
279+
280+
if (storage.length() == 0) {
281+
builder.append("[]");
282+
} else {
283+
startRecursion(encoder, list);
284+
285+
builder.append('[');
286+
if (encoder.indent != PNone.NONE) {
287+
indentLevel++;
288+
}
289+
290+
for (int i = 0; i < storage.length(); i++) {
291+
if (i > 0) {
292+
builder.append(encoder.itemSeparator);
293+
}
294+
appendListObj(encoder, builder, storage.getItemNormalized(i), indentLevel);
295+
}
296+
297+
builder.append(']');
298+
endRecursion(encoder, list);
299+
}
300+
}
301+
}
302+
}

0 commit comments

Comments
 (0)