Skip to content

Commit a29e6f6

Browse files
lukasstadlerlukasstadler
committed
[GR-9831] Unify code paths in parser, properly handle SyntaxError using traceback module.
PullRequest: graalpython/121
2 parents 52664d8 + a42aaa1 commit a29e6f6

File tree

23 files changed

+247
-398
lines changed

23 files changed

+247
-398
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,4 @@ Universal Permissive License v 1.0 as shown at
2222
[http://oss.oracle.com/licenses/upl](http://oss.oracle.com/licenses/upl). This
2323
implementation is in part derived from and contains additional code from 3rd
2424
parties, the copyrights and licensing of which is detailed in the
25-
[LICENSE](LICENSE) file.
25+
[LICENSE](LICENSE) and [3rd_party_licenses.txt](3rd_party_licenses.txt) files.

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/PythonTests.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,13 @@
5353

5454
import com.oracle.graal.python.PythonLanguage;
5555
import com.oracle.graal.python.runtime.PythonContext;
56-
import com.oracle.graal.python.runtime.PythonParseResult;
56+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
5757
import com.oracle.graal.python.runtime.exception.PException;
5858
import com.oracle.graal.python.test.interop.JavaInteropTest;
5959
import com.oracle.truffle.api.Truffle;
6060
import com.oracle.truffle.api.frame.FrameDescriptor;
6161
import com.oracle.truffle.api.frame.VirtualFrame;
62+
import com.oracle.truffle.api.nodes.RootNode;
6263
import com.oracle.truffle.api.source.MissingMIMETypeException;
6364
import com.oracle.truffle.api.source.MissingNameException;
6465
import com.oracle.truffle.api.source.Source;
@@ -260,15 +261,15 @@ private static String getFileContent(File file) {
260261
return ret;
261262
}
262263

263-
public static PythonParseResult getParseResult(com.oracle.truffle.api.source.Source source, PrintStream out, PrintStream err) {
264+
public static RootNode getParseResult(com.oracle.truffle.api.source.Source source, PrintStream out, PrintStream err) {
264265
PythonTests.ensureContext();
265266
PythonContext ctx = PythonLanguage.getContext();
266267
ctx.setOut(out);
267268
ctx.setErr(err);
268-
return ctx.getCore().getParser().parse(ctx.getCore(), source);
269+
return (RootNode) ctx.getCore().getParser().parse(ParserMode.File, ctx.getCore(), source, null);
269270
}
270271

271-
public static PythonParseResult getParseResult(String code) {
272+
public static RootNode getParseResult(String code) {
272273
final ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
273274
Builder<RuntimeException, MissingMIMETypeException, MissingNameException> newBuilder = Source.newBuilder(code);
274275
newBuilder.mimeType(PythonLanguage.MIME_TYPE);

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/generator/GeneratorExpressionTranslationTests.java

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333

3434
import com.oracle.graal.python.nodes.function.FunctionDefinitionNode;
3535
import com.oracle.graal.python.nodes.function.GeneratorExpressionNode;
36-
import com.oracle.graal.python.runtime.PythonParseResult;
3736
import com.oracle.truffle.api.nodes.Node;
3837
import com.oracle.truffle.api.nodes.NodeUtil;
3938
import com.oracle.truffle.api.nodes.RootNode;
@@ -57,8 +56,8 @@ public void generatorExpressionAsIterator() {
5756
" for i in (x for x in range(n)):\n" + //
5857
" item = i\n" +
5958
" print(item)";
60-
PythonParseResult parsed = getParseResult(source);
61-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
59+
RootNode parsed = getParseResult(source);
60+
RootNode root = getFunctionRoot(parsed, "foo");
6261
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
6362
assertTrue(genexp == 1);
6463
}
@@ -70,8 +69,8 @@ public void generatorExpressionAsArgumentToConstructor() {
7069
" n = 5\n" + //
7170
" return list(x for x in range(n))\n";
7271

73-
PythonParseResult parsed = getParseResult(source);
74-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
72+
RootNode parsed = getParseResult(source);
73+
RootNode root = getFunctionRoot(parsed, "foo");
7574
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
7675
assertTrue(genexp != 0);
7776
}
@@ -84,8 +83,8 @@ public void assignedToLocalVar() {
8483
" ll = (x for x in range(n))\n" + //
8584
" return list(ll)\n";
8685

87-
PythonParseResult parsed = getParseResult(source);
88-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
86+
RootNode parsed = getParseResult(source);
87+
RootNode root = getFunctionRoot(parsed, "foo");
8988
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
9089
assertTrue(genexp != 0);
9190
}
@@ -98,8 +97,8 @@ public void escapeByReturn() {
9897
" ll = (x for x in range(n))\n" + //
9998
" return ll\n";
10099

101-
PythonParseResult parsed = getParseResult(source);
102-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
100+
RootNode parsed = getParseResult(source);
101+
RootNode root = getFunctionRoot(parsed, "foo");
103102
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
104103
assertTrue(genexp != 0);
105104
}
@@ -113,8 +112,8 @@ public void escapeByStore() {
113112
" ll = (x for x in range(n))\n" + //
114113
" LIST[0] = ll\n";
115114

116-
PythonParseResult parsed = getParseResult(source);
117-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
115+
RootNode parsed = getParseResult(source);
116+
RootNode root = getFunctionRoot(parsed, "foo");
118117
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
119118
assertTrue(genexp != 0);
120119
}
@@ -127,8 +126,8 @@ public void escapeByCall() {
127126
" ll = (x for x in range(n))\n" + //
128127
" LIST.append(ll)\n";
129128

130-
PythonParseResult parsed = getParseResult(source);
131-
RootNode root = getFunctionRoot(parsed.getRootNode(), "foo");
129+
RootNode parsed = getParseResult(source);
130+
RootNode root = getFunctionRoot(parsed, "foo");
132131
int genexp = NodeUtil.findAllNodeInstances(root, GeneratorExpressionNode.class).size();
133132
assertTrue(genexp != 0);
134133
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/grammar/TestParserTranslator.java

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -99,10 +99,11 @@
9999
import com.oracle.graal.python.nodes.subscript.GetItemNode;
100100
import com.oracle.graal.python.nodes.subscript.SetItemNode;
101101
import com.oracle.graal.python.runtime.PythonContext;
102-
import com.oracle.graal.python.runtime.PythonParseResult;
102+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
103103
import com.oracle.graal.python.test.PythonTests;
104104
import com.oracle.truffle.api.nodes.Node;
105105
import com.oracle.truffle.api.nodes.NodeUtil;
106+
import com.oracle.truffle.api.nodes.RootNode;
106107
import com.oracle.truffle.api.source.Source;
107108

108109
public class TestParserTranslator {
@@ -112,15 +113,15 @@ public TestParserTranslator() {
112113
context = PythonTests.getContext();
113114
}
114115

115-
PythonParseResult parse(String src) {
116+
RootNode parse(String src) {
116117
Source source = Source.newBuilder(src).mimeType(PythonLanguage.MIME_TYPE).name("foo").build();
117-
return context.getCore().getParser().parse(context.getCore(), source);
118+
return (RootNode) context.getCore().getParser().parse(ParserMode.File, context.getCore(), source, null);
118119
}
119120

120121
String parseToString(String src) {
121-
PythonParseResult result = parse(src);
122+
RootNode result = parse(src);
122123
ByteArrayOutputStream out = new ByteArrayOutputStream();
123-
NodeUtil.printCompactTree(out, result.getRootNode());
124+
NodeUtil.printCompactTree(out, result);
124125
return out.toString();
125126
}
126127

@@ -141,10 +142,6 @@ <T> T getFirstChild(Node result, Class<? extends T> klass) {
141142
return getChild(result, 0, klass);
142143
}
143144

144-
<T> T getFirstChild(PythonParseResult result, Class<? extends T> klass) {
145-
return getFirstChild(result.getRootNode(), klass);
146-
}
147-
148145
<T> T parseAs(String src, Class<? extends T> klass) {
149146
return getFirstChild(parse(src), klass);
150147
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@
4848
import com.oracle.graal.python.runtime.PythonContext;
4949
import com.oracle.graal.python.runtime.PythonCore;
5050
import com.oracle.graal.python.runtime.PythonOptions;
51-
import com.oracle.graal.python.runtime.PythonParseResult;
51+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
52+
import com.oracle.graal.python.runtime.exception.PException;
5253
import com.oracle.truffle.api.CallTarget;
5354
import com.oracle.truffle.api.CompilerAsserts;
5455
import com.oracle.truffle.api.CompilerDirectives;
@@ -248,18 +249,24 @@ protected CallTarget parse(ParsingRequest request) throws Exception {
248249
if (!pythonCore.isInitialized()) {
249250
pythonCore.initialize();
250251
}
251-
context.initializeMainModule(request.getSource().getPath());
252+
Source source = request.getSource();
253+
context.initializeMainModule(source.getPath());
252254

253255
// if we are running the interpreter, module 'site' is automatically imported
254-
if (request.getSource().isInteractive()) {
256+
if (source.isInteractive()) {
255257
CompilerAsserts.neverPartOfCompilation();
256258
// no frame required
257259
new ImportNode("site").execute(null);
258260
}
259-
PythonParseResult parseResult = pythonCore.getParser().parse(pythonCore, request.getSource());
260-
RootNode root = parseResult.getRootNode();
261-
root = new TopLevelExceptionHandler(this, root);
262-
return Truffle.getRuntime().createCallTarget(root);
261+
RootNode root;
262+
try {
263+
root = (RootNode) pythonCore.getParser().parse(source.isInteractive() ? ParserMode.InteractiveStatement : ParserMode.File, pythonCore, source, null);
264+
} catch (PException e) {
265+
// handle PException during parsing (PIncompleteSourceException will propagate through)
266+
Truffle.getRuntime().createCallTarget(new TopLevelExceptionHandler(this, e)).call();
267+
throw e;
268+
}
269+
return Truffle.getRuntime().createCallTarget(new TopLevelExceptionHandler(this, root));
263270
}
264271

265272
@Override
@@ -310,7 +317,7 @@ private Object parseAndEval(PythonContext context, MaterializedFrame frame) {
310317
@TruffleBoundary
311318
protected static PNode parseInline(Source code, PythonContext context, MaterializedFrame lexicalContextFrame) {
312319
PythonCore pythonCore = context.getCore();
313-
return pythonCore.getParser().parseInline(pythonCore, code, lexicalContextFrame);
320+
return (PNode) pythonCore.getParser().parse(ParserMode.InlineEvaluation, pythonCore, code, lexicalContextFrame);
314321
}
315322

316323
@Override

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434

3535
import java.io.IOException;
3636
import java.net.MalformedURLException;
37-
import java.net.URI;
3837
import java.net.URL;
3938
import java.util.Arrays;
4039
import java.util.HashMap;
@@ -94,9 +93,9 @@
9493
import com.oracle.graal.python.builtins.objects.floats.FloatBuiltins;
9594
import com.oracle.graal.python.builtins.objects.foreign.TruffleObjectBuiltins;
9695
import com.oracle.graal.python.builtins.objects.frame.FrameBuiltins;
96+
import com.oracle.graal.python.builtins.objects.function.AbstractFunctionBuiltins;
9797
import com.oracle.graal.python.builtins.objects.function.BuiltinFunctionBuiltins;
9898
import com.oracle.graal.python.builtins.objects.function.FunctionBuiltins;
99-
import com.oracle.graal.python.builtins.objects.function.AbstractFunctionBuiltins;
10099
import com.oracle.graal.python.builtins.objects.function.PArguments;
101100
import com.oracle.graal.python.builtins.objects.function.PBuiltinFunction;
102101
import com.oracle.graal.python.builtins.objects.function.PFunction;
@@ -134,8 +133,8 @@
134133
import com.oracle.graal.python.runtime.PythonContext;
135134
import com.oracle.graal.python.runtime.PythonCore;
136135
import com.oracle.graal.python.runtime.PythonOptions;
137-
import com.oracle.graal.python.runtime.PythonParseResult;
138136
import com.oracle.graal.python.runtime.PythonParser;
137+
import com.oracle.graal.python.runtime.PythonParser.ParserMode;
139138
import com.oracle.graal.python.runtime.exception.PException;
140139
import com.oracle.graal.python.runtime.exception.PythonErrorType;
141140
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
@@ -147,6 +146,7 @@
147146
import com.oracle.truffle.api.TruffleLanguage.Env;
148147
import com.oracle.truffle.api.interop.TruffleObject;
149148
import com.oracle.truffle.api.nodes.Node;
149+
import com.oracle.truffle.api.nodes.RootNode;
150150
import com.oracle.truffle.api.source.Source;
151151

152152
/**
@@ -612,7 +612,7 @@ public void exportCInterface(PythonContext context) {
612612
env.exportSymbol("python_builtins", builtinsModule);
613613

614614
// export all exception classes for the C API
615-
for (PythonErrorType errorType : PythonErrorType.values()) {
615+
for (PythonErrorType errorType : PythonErrorType.VALUES) {
616616
PythonClass errorClass = getErrorClass(errorType);
617617
env.exportSymbol("python_" + errorClass.getName(), errorClass);
618618
}
@@ -780,26 +780,25 @@ private static Source getSource(String basename, String prefix) {
780780
try {
781781
return env.newSourceBuilder(file).name(basename).mimeType(PythonLanguage.MIME_TYPE).build();
782782
} catch (SecurityException | IOException t) {
783-
// TODO: XXX: This will only work if we already have cached parse trees
784-
return Source.newBuilder("").uri(URI.create(file.getPath())).name(basename).mimeType(PythonLanguage.MIME_TYPE).build();
783+
throw new RuntimeException("Could not read core library from " + file);
785784
}
786785
}
787786
}
788787

789788
private void loadFile(String s, String prefix) {
790-
PythonParseResult parsedModule = getParser().parse(this, getSource(s, prefix));
789+
RootNode parsedModule = (RootNode) getParser().parse(ParserMode.File, this, getSource(s, prefix), null);
791790
PythonModule mod = lookupBuiltinModule(s);
792791
if (mod == null) {
793792
// use an anonymous module for the side-effects
794793
mod = factory().createPythonModule("__anonymous__");
795794
}
796-
CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule.getRootNode());
795+
CallTarget callTarget = Truffle.getRuntime().createCallTarget(parsedModule);
797796
callTarget.call(PArguments.withGlobals(mod));
798797
}
799798

800799
private void findKnownExceptionTypes() {
801-
errorClasses = new PythonClass[PythonErrorType.values().length];
802-
for (PythonErrorType type : PythonErrorType.values()) {
800+
errorClasses = new PythonClass[PythonErrorType.VALUES.length];
801+
for (PythonErrorType type : PythonErrorType.VALUES) {
803802
errorClasses[type.ordinal()] = (PythonClass) builtinsModule.getAttribute(type.name());
804803
}
805804
}

0 commit comments

Comments
 (0)