Skip to content

Commit 9330305

Browse files
lukasstadlerlukasstadler
committed
unify code paths in parser, properly handle SyntaxError using traceback module
1 parent c54567c commit 9330305

File tree

20 files changed

+236
-383
lines changed

20 files changed

+236
-383
lines changed

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/PythonTests.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,13 @@
5353

5454
import com.oracle.graal.python.PythonLanguage;
5555
import com.oracle.graal.python.runtime.PythonContext;
56-
import com.oracle.graal.python.runtime.PythonParseResult;
56+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
5757
import com.oracle.graal.python.runtime.exception.PException;
5858
import com.oracle.graal.python.test.interop.JavaInteropTest;
5959
import com.oracle.truffle.api.Truffle;
6060
import com.oracle.truffle.api.frame.FrameDescriptor;
6161
import com.oracle.truffle.api.frame.VirtualFrame;
62+
import com.oracle.truffle.api.nodes.RootNode;
6263
import com.oracle.truffle.api.source.MissingMIMETypeException;
6364
import com.oracle.truffle.api.source.MissingNameException;
6465
import com.oracle.truffle.api.source.Source;
@@ -260,15 +261,15 @@ private static String getFileContent(File file) {
260261
return ret;
261262
}
262263

263-
public static PythonParseResult getParseResult(com.oracle.truffle.api.source.Source source, PrintStream out, PrintStream err) {
264+
public static RootNode getParseResult(com.oracle.truffle.api.source.Source source, PrintStream out, PrintStream err) {
264265
PythonTests.ensureContext();
265266
PythonContext ctx = PythonLanguage.getContext();
266267
ctx.setOut(out);
267268
ctx.setErr(err);
268-
return ctx.getCore().getParser().parse(ctx.getCore(), source);
269+
return (RootNode) ctx.getCore().getParser().parse(ParserMode.File, ctx.getCore(), source, null);
269270
}
270271

271-
public static PythonParseResult getParseResult(String code) {
272+
public static RootNode getParseResult(String code) {
272273
final ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
273274
Builder<RuntimeException, MissingMIMETypeException, MissingNameException> newBuilder = Source.newBuilder(code);
274275
newBuilder.mimeType(PythonLanguage.MIME_TYPE);

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/generator/GeneratorExpressionTranslationTests.java

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333

3434
import com.oracle.graal.python.nodes.function.FunctionDefinitionNode;
3535
import com.oracle.graal.python.nodes.function.GeneratorExpressionNode;
36-
import com.oracle.graal.python.runtime.PythonParseResult;
3736
import com.oracle.truffle.api.nodes.Node;
3837
import com.oracle.truffle.api.nodes.NodeUtil;
3938
import com.oracle.truffle.api.nodes.RootNode;
@@ -57,8 +56,8 @@ public void generatorExpressionAsIterator() {
5756
" for i in (x for x in range(n)):\n" + //
5857
" item = i\n" +
5958
" print(item)";
60-
PythonParseResult parsed = getParseResult(source);
61-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
59+
RootNode parsed = getParseResult(source);
60+
RootNode root = getFunctionRoot(parsed, "foo");
6261
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
6362
assertTrue(genexp == 1);
6463
}
@@ -70,8 +69,8 @@ public void generatorExpressionAsArgumentToConstructor() {
7069
" n = 5\n" + //
7170
" return list(x for x in range(n))\n";
7271

73-
PythonParseResult parsed = getParseResult(source);
74-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
72+
RootNode parsed = getParseResult(source);
73+
RootNode root = getFunctionRoot(parsed, "foo");
7574
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
7675
assertTrue(genexp != 0);
7776
}
@@ -84,8 +83,8 @@ public void assignedToLocalVar() {
8483
" ll = (x for x in range(n))\n" + //
8584
" return list(ll)\n";
8685

87-
PythonParseResult parsed = getParseResult(source);
88-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
86+
RootNode parsed = getParseResult(source);
87+
RootNode root = getFunctionRoot(parsed, "foo");
8988
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
9089
assertTrue(genexp != 0);
9190
}
@@ -98,8 +97,8 @@ public void escapeByReturn() {
9897
" ll = (x for x in range(n))\n" + //
9998
" return ll\n";
10099

101-
PythonParseResult parsed = getParseResult(source);
102-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
100+
RootNode parsed = getParseResult(source);
101+
RootNode root = getFunctionRoot(parsed, "foo");
103102
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
104103
assertTrue(genexp != 0);
105104
}
@@ -113,8 +112,8 @@ public void escapeByStore() {
113112
" ll = (x for x in range(n))\n" + //
114113
" LIST[0] = ll\n";
115114

116-
PythonParseResult parsed = getParseResult(source);
117-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
115+
RootNode parsed = getParseResult(source);
116+
RootNode root = getFunctionRoot(parsed, "foo");
118117
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
119118
assertTrue(genexp != 0);
120119
}
@@ -127,8 +126,8 @@ public void escapeByCall() {
127126
" ll = (x for x in range(n))\n" + //
128127
" LIST.append(ll)\n";
129128

130-
PythonParseResult parsed = getParseResult(source);
131-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
129+
RootNode parsed = getParseResult(source);
130+
RootNode root = getFunctionRoot(parsed, "foo");
132131
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
133132
assertTrue(genexp != 0);
134133
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/grammar/TestParserTranslator.java

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -99,10 +99,11 @@
9999
import com.oracle.graal.python.nodes.subscript.GetItemNode;
100100
import com.oracle.graal.python.nodes.subscript.SetItemNode;
101101
import com.oracle.graal.python.runtime.PythonContext;
102-
import com.oracle.graal.python.runtime.PythonParseResult;
102+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
103103
import com.oracle.graal.python.test.PythonTests;
104104
import com.oracle.truffle.api.nodes.Node;
105105
import com.oracle.truffle.api.nodes.NodeUtil;
106+
import com.oracle.truffle.api.nodes.RootNode;
106107
import com.oracle.truffle.api.source.Source;
107108

108109
public class TestParserTranslator {
@@ -112,15 +113,15 @@ public TestParserTranslator() {
112113
context = PythonTests.getContext();
113114
}
114115

115-
PythonParseResult parse(String src) {
116+
RootNode parse(String src) {
116117
Source source = Source.newBuilder(src).mimeType(PythonLanguage.MIME_TYPE).name("foo").build();
117-
return context.getCore().getParser().parse(context.getCore(), source);
118+
return (RootNode) context.getCore().getParser().parse(ParserMode.File, context.getCore(), source, null);
118119
}
119120

120121
String parseToString(String src) {
121-
PythonParseResult result = parse(src);
122+
RootNode result = parse(src);
122123
ByteArrayOutputStream out = new ByteArrayOutputStream();
123-
NodeUtil.printCompactTree(out, result.getRootNode());
124+
NodeUtil.printCompactTree(out, result);
124125
return out.toString();
125126
}
126127

@@ -141,10 +142,6 @@ <T> T getFirstChild(Node result, Class<? extends T> klass) {
141142
return getChild(result, 0, klass);
142143
}
143144

144-
<T> T getFirstChild(PythonParseResult result, Class<? extends T> klass) {
145-
return getFirstChild(result.getRootNode(), klass);
146-
}
147-
148145
<T> T parseAs(String src, Class<? extends T> klass) {
149146
return getFirstChild(parse(src), klass);
150147
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@
4848
import com.oracle.graal.python.runtime.PythonContext;
4949
import com.oracle.graal.python.runtime.PythonCore;
5050
import com.oracle.graal.python.runtime.PythonOptions;
51-
import com.oracle.graal.python.runtime.PythonParseResult;
51+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
52+
import com.oracle.graal.python.runtime.exception.PException;
5253
import com.oracle.truffle.api.CallTarget;
5354
import com.oracle.truffle.api.CompilerAsserts;
5455
import com.oracle.truffle.api.CompilerDirectives;
@@ -248,18 +249,24 @@ protected CallTarget parse(ParsingRequest request) throws Exception {
248249
if (!pythonCore.isInitialized()) {
249250
pythonCore.initialize();
250251
}
251-
context.initializeMainModule(request.getSource().getPath());
252+
Source source = request.getSource();
253+
context.initializeMainModule(source.getPath());
252254

253255
// if we are running the interpreter, module 'site' is automatically imported
254-
if (request.getSource().isInteractive()) {
256+
if (source.isInteractive()) {
255257
CompilerAsserts.neverPartOfCompilation();
256258
// no frame required
257259
new ImportNode("site").execute(null);
258260
}
259-
PythonParseResult parseResult = pythonCore.getParser().parse(pythonCore, request.getSource());
260-
RootNode root = parseResult.getRootNode();
261-
root = new TopLevelExceptionHandler(this, root);
262-
return Truffle.getRuntime().createCallTarget(root);
261+
RootNode root;
262+
try {
263+
root = (RootNode) pythonCore.getParser().parse(source.isInteractive() ? ParserMode.InteractiveStatement : ParserMode.File, pythonCore, source, null);
264+
} catch (PException e) {
265+
// handle PException during parsing (PIncompleteSourceException will propagate through)
266+
Truffle.getRuntime().createCallTarget(new TopLevelExceptionHandler(this, e)).call();
267+
throw e;
268+
}
269+
return Truffle.getRuntime().createCallTarget(new TopLevelExceptionHandler(this, root));
263270
}
264271

265272
@Override
@@ -310,7 +317,7 @@ private Object parseAndEval(PythonContext context, MaterializedFrame frame) {
310317
@TruffleBoundary
311318
protected static PNode parseInline(Source code, PythonContext context, MaterializedFrame lexicalContextFrame) {
312319
PythonCore pythonCore = context.getCore();
313-
return pythonCore.getParser().parseInline(pythonCore, code, lexicalContextFrame);
320+
return (PNode) pythonCore.getParser().parse(ParserMode.InlineEvaluation, pythonCore, code, lexicalContextFrame);
314321
}
315322

316323
@Override

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,9 @@
9494
import com.oracle.graal.python.builtins.objects.floats.FloatBuiltins;
9595
import com.oracle.graal.python.builtins.objects.foreign.TruffleObjectBuiltins;
9696
import com.oracle.graal.python.builtins.objects.frame.FrameBuiltins;
97+
import com.oracle.graal.python.builtins.objects.function.AbstractFunctionBuiltins;
9798
import com.oracle.graal.python.builtins.objects.function.BuiltinFunctionBuiltins;
9899
import com.oracle.graal.python.builtins.objects.function.FunctionBuiltins;
99-
import com.oracle.graal.python.builtins.objects.function.AbstractFunctionBuiltins;
100100
import com.oracle.graal.python.builtins.objects.function.PArguments;
101101
import com.oracle.graal.python.builtins.objects.function.PBuiltinFunction;
102102
import com.oracle.graal.python.builtins.objects.function.PFunction;
@@ -134,8 +134,8 @@
134134
import com.oracle.graal.python.runtime.PythonContext;
135135
import com.oracle.graal.python.runtime.PythonCore;
136136
import com.oracle.graal.python.runtime.PythonOptions;
137-
import com.oracle.graal.python.runtime.PythonParseResult;
138137
import com.oracle.graal.python.runtime.PythonParser;
138+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
139139
import com.oracle.graal.python.runtime.exception.PException;
140140
import com.oracle.graal.python.runtime.exception.PythonErrorType;
141141
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
@@ -147,6 +147,7 @@
147147
import com.oracle.truffle.api.TruffleLanguage.Env;
148148
import com.oracle.truffle.api.interop.TruffleObject;
149149
import com.oracle.truffle.api.nodes.Node;
150+
import com.oracle.truffle.api.nodes.RootNode;
150151
import com.oracle.truffle.api.source.Source;
151152

152153
/**
@@ -612,7 +613,7 @@ public void exportCInterface(PythonContext context) {
612613
env.exportSymbol("python_builtins", builtinsModule);
613614

614615
// export all exception classes for the C API
615-
for (PythonErrorType errorType : PythonErrorType.values()) {
616+
for (PythonErrorType errorType : PythonErrorType.VALUES) {
616617
PythonClass errorClass = getErrorClass(errorType);
617618
env.exportSymbol("python_" + errorClass.getName(), errorClass);
618619
}
@@ -787,19 +788,19 @@ private static Source getSource(String basename, String prefix) {
787788
}
788789

789790
private void loadFile(String s, String prefix) {
790-
PythonParseResult parsedModule = getParser().parse(this, getSource(s, prefix));
791+
RootNode parsedModule = (RootNode) getParser().parse(ParserMode.File, this, getSource(s, prefix), null);
791792
PythonModule mod = lookupBuiltinModule(s);
792793
if (mod == null) {
793794
// use an anonymous module for the side-effects
794795
mod = factory().createPythonModule("__anonymous__");
795796
}
796-
CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule.getRootNode());
797+
CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule);
797798
callTarget.call(PArguments.withGlobals(mod));
798799
}
799800

800801
private void findKnownExceptionTypes() {
801-
errorClasses = new PythonClass[PythonErrorType.values().length];
802-
for (PythonErrorType type : PythonErrorType.values()) {
802+
errorClasses = new PythonClass[PythonErrorType.VALUES.length];
803+
for (PythonErrorType type : PythonErrorType.VALUES) {
803804
errorClasses[type.ordinal()] = (PythonClass) builtinsModule.getAttribute(type.name());
804805
}
805806
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,8 @@
123123
import com.oracle.graal.python.runtime.PythonContext;
124124
import com.oracle.graal.python.runtime.PythonCore;
125125
import com.oracle.graal.python.runtime.PythonOptions;
126-
import com.oracle.graal.python.runtime.PythonParseResult;
127126
import com.oracle.graal.python.runtime.PythonParser;
127+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
128128
import com.oracle.graal.python.runtime.exception.PException;
129129
import com.oracle.graal.python.runtime.exception.PythonErrorType;
130130
import com.oracle.truffle.api.CallTarget;
@@ -429,25 +429,27 @@ public Object eval(VirtualFrame frame, String expression, @SuppressWarnings("unu
429429
Frame callerFrame = readCallerFrameNode.executeWith(frame);
430430
PythonObject callerGlobals = PArguments.getGlobals(callerFrame);
431431
PCell[] callerClosure = PArguments.getClosure(callerFrame);
432-
return evalExpression(expression, callerGlobals, callerGlobals, callerClosure);
432+
return evalExpression(expression, callerGlobals, callerGlobals, callerClosure, callerFrame);
433433
}
434434

435435
@Specialization
436-
public Object eval(String expression, PythonObject globals, @SuppressWarnings("unused") PNone locals) {
437-
return evalExpression(expression, globals, globals, null);
436+
public Object eval(VirtualFrame frame, String expression, PythonObject globals, @SuppressWarnings("unused") PNone locals) {
437+
Frame callerFrame = readCallerFrameNode.executeWith(frame);
438+
return evalExpression(expression, globals, globals, null, callerFrame);
438439
}
439440

440441
@Specialization
441-
public Object eval(String expression, PythonObject globals, PythonObject locals) {
442-
return evalExpression(expression, globals, locals, null);
442+
public Object eval(VirtualFrame frame, String expression, PythonObject globals, PythonObject locals) {
443+
Frame callerFrame = readCallerFrameNode.executeWith(frame);
444+
return evalExpression(expression, globals, locals, null, callerFrame);
443445
}
444446

445447
@Specialization
446448
public Object eval(VirtualFrame frame, String expression, @SuppressWarnings("unused") PNone globals, PythonObject locals) {
447449
Frame callerFrame = readCallerFrameNode.executeWith(frame);
448450
PythonObject callerGlobals = PArguments.getGlobals(callerFrame);
449451
PCell[] callerClosure = PArguments.getClosure(callerFrame);
450-
return evalExpression(expression, callerGlobals, locals, callerClosure);
452+
return evalExpression(expression, callerGlobals, locals, callerClosure, callerFrame);
451453
}
452454

453455
@Specialization
@@ -487,7 +489,7 @@ private static Object evalExpression(PCode code, PythonObject globals, PythonObj
487489
}
488490

489491
@TruffleBoundary
490-
private Object evalExpression(String expression, PythonObject globals, PythonObject locals, PCell[] closure) {
492+
private Object evalExpression(String expression, PythonObject globals, PythonObject locals, PCell[] closure, Frame callerFrame) {
491493
String name = "<eval>";
492494
if (globals instanceof PDict) {
493495
Object nameObject = getNameNode.execute(globals, __NAME__);
@@ -496,8 +498,9 @@ private Object evalExpression(String expression, PythonObject globals, PythonObj
496498
}
497499
}
498500
PythonParser parser = getCore().getParser();
499-
PythonParseResult parsed = parser.parseEval(getCore(), expression, name);
500-
return evalNode(parsed.getRootNode(), globals, locals, closure);
501+
Source source = Source.newBuilder(expression).name(name).mimeType(PythonLanguage.MIME_TYPE).build();
502+
RootNode parsed = (RootNode) parser.parse(ParserMode.Eval, getCore(), source, callerFrame);
503+
return evalNode(parsed, globals, locals, closure);
501504
}
502505

503506
/**
@@ -527,19 +530,19 @@ Object compile(PBytes source, String filename, String mode, Object kwFlags, Obje
527530
@SuppressWarnings("unused")
528531
@Specialization
529532
@TruffleBoundary
530-
Object compile(String source, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize) {
531-
PythonParser parser = getCore().getParser();
532-
PythonParseResult result;
533+
Object compile(String expression, String filename, String mode, Object kwFlags, Object kwDontInherit, Object kwOptimize) {
534+
Source source = Source.newBuilder(expression).name(filename).mimeType(PythonLanguage.MIME_TYPE).build();
535+
ParserMode pm;
533536
if (mode.equals("exec")) {
534-
result = parser.parseExec(getCore(), source, filename);
537+
pm = ParserMode.File;
535538
} else if (mode.equals("eval")) {
536-
result = parser.parseEval(getCore(), source, filename);
539+
pm = ParserMode.Eval;
537540
} else if (mode.equals("single")) {
538-
result = parser.parseSingle(getCore(), source, filename);
541+
pm = ParserMode.Statement;
539542
} else {
540543
throw raise(ValueError, "compile() mode must be 'exec', 'eval' or 'single'");
541544
}
542-
return factory().createCode(result);
545+
return factory().createCode((RootNode) getCore().getParser().parse(pm, getCore(), source, null));
543546
}
544547

545548
@SuppressWarnings("unused")
@@ -1227,8 +1230,8 @@ public abstract static class LoadBuiltinsNode extends PythonBuiltinNode {
12271230
public Object doIt(String name) {
12281231
PythonModule mod = getCore().isInitialized() ? getContext().getBuiltins() : getCore().lookupBuiltinModule("builtins");
12291232
Source src = getCore().getCoreSource(name);
1230-
PythonParseResult parsedModule = getCore().getParser().parse(getCore(), src);
1231-
CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule.getRootNode());
1233+
RootNode parsedModule = (RootNode) getCore().getParser().parse(ParserMode.File, getCore(), src, null);
1234+
CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule);
12321235
callTarget.call(PArguments.withGlobals(mod));
12331236
return PNone.NONE;
12341237
}

0 commit comments

Comments
 (0)