Skip to content

Commit 8550cbd

Browse files
ppislppisl
committed
[GR-24297] Python does not support parsing with arguments api.
1 parent fc9cf11 commit 8550cbd

File tree

14 files changed

+1988
-1540
lines changed

14 files changed

+1988
-1540
lines changed

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/PythonTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,7 @@ public static RootNode getParseResult(com.oracle.truffle.api.source.Source sourc
294294
PythonContext ctx = PythonLanguage.getContext();
295295
ctx.setOut(out);
296296
ctx.setErr(err);
297-
return (RootNode) ctx.getCore().getParser().parse(ParserMode.File, ctx.getCore(), source, null);
297+
return (RootNode) ctx.getCore().getParser().parse(ParserMode.File, ctx.getCore(), source, null, null);
298298
}
299299

300300
public static RootNode getParseResult(String code) {

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParseWithArgumentsTests.java

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.test.parser;
42+
43+
import com.oracle.graal.python.PythonLanguage;
44+
import com.oracle.graal.python.builtins.objects.PNone;
45+
import com.oracle.truffle.api.CallTarget;
46+
import com.oracle.truffle.api.source.Source;
47+
import static org.junit.Assert.assertEquals;
48+
import org.junit.Test;
49+
50+
public class ParseWithArgumentsTests extends ParserTestBase {
51+
52+
@Test
53+
public void testSimpple01() throws Exception {
54+
Source source = createSource("arg1");
55+
CallTarget target = context.getEnv().parsePublic(source, "arg1");
56+
assertEquals(66, target.call(66));
57+
assertEquals(false, target.call(false));
58+
assertEquals("Ahoj", target.call("Ahoj"));
59+
}
60+
61+
@Test
62+
public void testSimpple02() throws Exception {
63+
Source source = createSource("arg1 + arg2");
64+
CallTarget target = context.getEnv().parsePublic(source, "arg1", "arg2");
65+
assertEquals(11, target.call(5, 6));
66+
assertEquals("AhojHello", target.call("Ahoj", "Hello"));
67+
}
68+
69+
@Test
70+
public void testMoreStatements() throws Exception {
71+
Source source = createSource("tmp = arg1 + arg2\n" + "2 * tmp");
72+
CallTarget target = context.getEnv().parsePublic(source, "arg1", "arg2");
73+
assertEquals(22, target.call(5, 6));
74+
assertEquals("AhojHelloAhojHello", target.call("Ahoj", "Hello"));
75+
}
76+
77+
@Test
78+
public void testReturnStatement() throws Exception {
79+
Source source = createSource("tmp = arg1 + arg2\n" + "return 2 * tmp");
80+
CallTarget target = context.getEnv().parsePublic(source, "arg1", "arg2");
81+
assertEquals(22, target.call(5, 6));
82+
assertEquals("AhojHelloAhojHello", target.call("Ahoj", "Hello"));
83+
}
84+
85+
@Test
86+
public void testWitouthReturn() throws Exception {
87+
Source source = createSource("tmp = arg1 + arg2\n");
88+
CallTarget target = context.getEnv().parsePublic(source, "arg1", "arg2");
89+
assertEquals(PNone.NONE, target.call(5, 6));
90+
assertEquals(PNone.NONE, target.call("Ahoj", "Hello"));
91+
}
92+
93+
@Test
94+
public void testCompareWithAndWithouthArguments() throws Exception {
95+
Source source = createSource("22");
96+
CallTarget targetWithout = context.getEnv().parsePublic(source);
97+
CallTarget targetWith = context.getEnv().parsePublic(source, "arg1");
98+
assertEquals(22, targetWithout.call());
99+
assertEquals(22, targetWith.call("Hello"));
100+
}
101+
102+
@Test
103+
public void testPrintBuiltin() throws Exception {
104+
Source source = createSource("print('Ahoj')");
105+
CallTarget target = context.getEnv().parsePublic(source, "arg1");
106+
assertEquals(PNone.NONE, target.call(5));
107+
}
108+
109+
@Test
110+
public void testObjectMethods() throws Exception {
111+
Source source = createSource("arg1.upper()");
112+
CallTarget target = context.getEnv().parsePublic(source, "arg1");
113+
assertEquals("AHOJ", target.call("ahoj"));
114+
}
115+
116+
@Test
117+
public void testAbsGlobal() throws Exception {
118+
Source source = createSource("abs(arg1)");
119+
CallTarget target = context.getEnv().parsePublic(source, "arg1");
120+
assertEquals(10, target.call(-10));
121+
}
122+
123+
private static Source createSource(String code) {
124+
return Source.newBuilder(PythonLanguage.ID, code, "test.py").build();
125+
}
126+
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParserTestBase.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ protected Source createSource(File testFile) throws Exception {
9595
public Node parse(String src, String moduleName, PythonParser.ParserMode mode, Frame fd) {
9696
Source source = Source.newBuilder(PythonLanguage.ID, src, moduleName).build();
9797
PythonParser parser = context.getCore().getParser();
98-
Node result = ((PythonParserImpl) parser).parseN(mode, context.getCore(), source, fd);
98+
Node result = ((PythonParserImpl) parser).parseN(mode, context.getCore(), source, fd, null);
9999
lastGlobalScope = ((PythonParserImpl) parser).getLastGlobaScope();
100100
return result;
101101
}
@@ -106,7 +106,7 @@ public Node parse(String src, String moduleName, PythonParser.ParserMode mode) {
106106

107107
public Node parse(Source source, PythonParser.ParserMode mode) {
108108
PythonParser parser = context.getCore().getParser();
109-
Node result = ((PythonParserImpl) parser).parseN(mode, context.getCore(), source, null);
109+
Node result = ((PythonParserImpl) parser).parseN(mode, context.getCore(), source, null, null);
110110
lastGlobalScope = ((PythonParserImpl) parser).getLastGlobaScope();
111111
lastSST = ((PythonParserImpl) parser).getLastSST();
112112
return result;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 48 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import com.oracle.graal.python.builtins.objects.object.PythonObject;
4848
import com.oracle.graal.python.nodes.BuiltinNames;
4949
import com.oracle.graal.python.nodes.NodeFactory;
50+
import com.oracle.graal.python.nodes.call.InvokeNode;
5051
import com.oracle.graal.python.nodes.control.TopLevelExceptionHandler;
5152
import com.oracle.graal.python.nodes.expression.ExpressionNode;
5253
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
@@ -60,6 +61,7 @@
6061
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
6162
import com.oracle.graal.python.util.Function;
6263
import com.oracle.graal.python.util.PFunctionArgsFinder;
64+
import com.oracle.graal.python.util.PythonUtils;
6365
import com.oracle.graal.python.util.Supplier;
6466
import com.oracle.truffle.api.Assumption;
6567
import com.oracle.truffle.api.CallTarget;
@@ -249,6 +251,9 @@ protected CallTarget parse(ParsingRequest request) {
249251
if (core.isInitialized()) {
250252
context.initializeMainModule(source.getPath());
251253
}
254+
if (!request.getArgumentNames().isEmpty()) {
255+
return Truffle.getRuntime().createCallTarget(parseWithArguments(request));
256+
}
252257
RootNode root = doParse(context, source);
253258
if (core.isInitialized()) {
254259
return Truffle.getRuntime().createCallTarget(new TopLevelExceptionHandler(this, root));
@@ -258,7 +263,7 @@ protected CallTarget parse(ParsingRequest request) {
258263
}
259264

260265
private RootNode doParse(PythonContext context, Source source) {
261-
ParserMode mode = null;
266+
ParserMode mode;
262267
if (source.isInteractive()) {
263268
if (context.getOption(PythonOptions.TerminalIsInteractive)) {
264269
// if we run through our own launcher, the sys.__displayhook__ would provide the
@@ -275,14 +280,54 @@ private RootNode doParse(PythonContext context, Source source) {
275280
}
276281
PythonCore pythonCore = context.getCore();
277282
try {
278-
return (RootNode) pythonCore.getParser().parse(mode, pythonCore, source, null);
283+
return (RootNode) pythonCore.getParser().parse(mode, pythonCore, source, null, null);
279284
} catch (PException e) {
280285
// handle PException during parsing (PIncompleteSourceException will propagate through)
281286
Truffle.getRuntime().createCallTarget(new TopLevelExceptionHandler(this, e)).call();
282287
throw e;
283288
}
284289
}
285290

291+
private RootNode parseWithArguments(ParsingRequest request) {
292+
final String[] argumentNames = request.getArgumentNames().toArray(new String[request.getArgumentNames().size()]);
293+
final Source source = request.getSource();
294+
CompilerDirectives.transferToInterpreter();
295+
final RootNode executableNode = new RootNode(this) {
296+
@Node.Child private RootNode rootNode;
297+
298+
protected Object[] preparePArguments(VirtualFrame frame) {
299+
int argumentsLength = frame.getArguments().length;
300+
Object[] arguments = PArguments.create(argumentsLength);
301+
PArguments.setGlobals(arguments, new PDict());
302+
PythonUtils.arraycopy(frame.getArguments(), 0, arguments, PArguments.USER_ARGUMENTS_OFFSET, argumentsLength);
303+
return arguments;
304+
}
305+
306+
@Override
307+
@TruffleBoundary
308+
public Object execute(VirtualFrame frame) {
309+
PythonContext context = lookupContextReference(PythonLanguage.class).get();
310+
assert context != null;
311+
if (!context.isInitialized()) {
312+
context.initialize();
313+
}
314+
if (rootNode == null) {
315+
CompilerDirectives.transferToInterpreterAndInvalidate();
316+
parse(context, frame);
317+
}
318+
Object[] args = preparePArguments(frame);
319+
Object result = InvokeNode.invokeUncached(rootNode.getCallTarget(), args);
320+
return result;
321+
}
322+
323+
private void parse(PythonContext context, VirtualFrame frame) {
324+
CompilerAsserts.neverPartOfCompilation();
325+
rootNode = (RootNode) context.getCore().getParser().parse(ParserMode.WithArguments, context.getCore(), source, frame, argumentNames);
326+
}
327+
};
328+
return executableNode;
329+
}
330+
286331
@Override
287332
protected ExecutableNode parse(InlineParsingRequest request) {
288333
CompilerDirectives.transferToInterpreter();
@@ -334,7 +379,7 @@ private Object parseAndEval(PythonContext context, MaterializedFrame frame) {
334379
@TruffleBoundary
335380
protected static ExpressionNode parseInline(Source code, PythonContext context, MaterializedFrame lexicalContextFrame) {
336381
PythonCore pythonCore = context.getCore();
337-
return (ExpressionNode) pythonCore.getParser().parse(ParserMode.InlineEvaluation, pythonCore, code, lexicalContextFrame);
382+
return (ExpressionNode) pythonCore.getParser().parse(ParserMode.InlineEvaluation, pythonCore, code, lexicalContextFrame, null);
338383
}
339384

340385
@Override

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ private Source getInternalSource(String basename, String prefix) {
692692
private void loadFile(String s, String prefix) {
693693
Supplier<CallTarget> getCode = () -> {
694694
Source source = getInternalSource(s, prefix);
695-
return Truffle.getRuntime().createCallTarget((RootNode) getParser().parse(ParserMode.File, this, source, null));
695+
return Truffle.getRuntime().createCallTarget((RootNode) getParser().parse(ParserMode.File, this, source, null, null));
696696
};
697697
RootCallTarget callTarget = (RootCallTarget) getLanguage().cacheCode(s, getCode);
698698
PythonModule mod = lookupBuiltinModule(s);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -764,7 +764,7 @@ PCode compile(String expression, String filename, String mode, Object kwFlags, O
764764
final String codeToCompile = code;
765765
Supplier<CallTarget> createCode = () -> {
766766
Source source = PythonLanguage.newSource(context, codeToCompile, filename, mayBeFromFile);
767-
return Truffle.getRuntime().createCallTarget((RootNode) getCore().getParser().parse(pm, getCore(), source, null));
767+
return Truffle.getRuntime().createCallTarget((RootNode) getCore().getParser().parse(pm, getCore(), source, null, null));
768768
};
769769
RootCallTarget ct;
770770
if (getCore().isInitialized()) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/literal/FormatStringLiteralNode.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ private static void raiseInvalidSyntax(FormatStringLiteralNode node, String mess
513513
private static ExpressionNode createExpression(String src, VirtualFrame frame) {
514514
PythonParser parser = PythonLanguage.getCore().getParser();
515515
Source source = Source.newBuilder(PythonLanguage.ID, src, "<fstring>").build();
516-
Node expression = parser.parse(PythonParser.ParserMode.InlineEvaluation, PythonLanguage.getCore(), source, frame);
516+
Node expression = parser.parse(PythonParser.ParserMode.InlineEvaluation, PythonLanguage.getCore(), source, frame, null);
517517
return (ExpressionNode) expression;
518518
}
519519
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonParserImpl.java

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import com.oracle.graal.python.parser.sst.BlockSSTNode;
5050
import com.oracle.graal.python.parser.sst.SSTDeserializer;
5151
import com.oracle.graal.python.parser.sst.SSTNode;
52+
import com.oracle.graal.python.parser.sst.SSTNodeUtils;
5253
import com.oracle.graal.python.parser.sst.SSTNodeWithScope;
5354
import com.oracle.graal.python.parser.sst.SSTNodeWithScopeFinder;
5455
import com.oracle.graal.python.parser.sst.SSTSerializerVisitor;
@@ -107,7 +108,7 @@ public byte[] serialize(RootNode rootNode) {
107108
if (!source.equals(lastParserResult.source)) {
108109
// we need to parse the source again
109110
PythonSSTNodeFactory sstFactory = new PythonSSTNodeFactory(PythonLanguage.getCore(), source);
110-
lastParserResult = parseWithANTLR(ParserMode.File, PythonLanguage.getCore(), sstFactory, source, null);
111+
lastParserResult = parseWithANTLR(ParserMode.File, PythonLanguage.getCore(), sstFactory, source, null, null);
111112
}
112113
try {
113114
dos.writeByte(SerializationUtils.VERSION);
@@ -221,7 +222,7 @@ public SSTNode getLastSST() {
221222
}
222223

223224
@Override
224-
public Node parse(ParserMode mode, ParserErrorCallback errors, Source source, Frame currentFrame) {
225+
public Node parse(ParserMode mode, ParserErrorCallback errors, Source source, Frame currentFrame, String[] argumentNames) {
225226
if (logFiles) {
226227
if (source.getPath() == null) {
227228
System.out.println("Parsing source without path " + source.getCharacters().length());
@@ -236,10 +237,10 @@ public Node parse(ParserMode mode, ParserErrorCallback errors, Source source, Fr
236237

237238
Node result;
238239
if (timeStatistics <= 0) {
239-
result = parseN(mode, errors, source, currentFrame);
240+
result = parseN(mode, errors, source, currentFrame, argumentNames);
240241
} else {
241242
long start = System.currentTimeMillis();
242-
result = parseN(mode, errors, source, currentFrame);
243+
result = parseN(mode, errors, source, currentFrame, argumentNames);
243244
long end = System.currentTimeMillis();
244245
if (timeStatistics > 0) {
245246
timeInParser = timeInParser + (end - start);
@@ -255,7 +256,7 @@ public Node parse(ParserMode mode, ParserErrorCallback errors, Source source, Fr
255256
return result;
256257
}
257258

258-
private CacheItem parseWithANTLR(ParserMode mode, ParserErrorCallback errors, PythonSSTNodeFactory sstFactory, Source source, Frame currentFrame) {
259+
private CacheItem parseWithANTLR(ParserMode mode, ParserErrorCallback errors, PythonSSTNodeFactory sstFactory, Source source, Frame currentFrame, String[] argumentNames) {
259260
FrameDescriptor inlineLocals = mode == ParserMode.InlineEvaluation ? currentFrame.getFrameDescriptor() : null;
260261
String sourceText = source.getCharacters().toString();
261262
// Preprocessing
@@ -292,6 +293,15 @@ private CacheItem parseWithANTLR(ParserMode mode, ParserErrorCallback errors, Py
292293
case Statement:
293294
parserSSTResult = parser.single_input(source.isInteractive(), inlineLocals).result;
294295
break;
296+
case WithArguments:
297+
// at the first, create global scope
298+
ScopeInfo globalScope = sstFactory.getScopeEnvironment().pushScope("module", ScopeInfo.ScopeKind.Module, currentFrame == null ? null : currentFrame.getFrameDescriptor());
299+
// we expect that the source is the body of the result function
300+
parserSSTResult = parser.withArguments_input(false, new FrameDescriptor()).result;
301+
// wrap the result with function definition
302+
ScopeInfo functionScope = globalScope.getFirstChildScope();
303+
parserSSTResult = SSTNodeUtils.createFunctionDefWithArguments(source.getName(), functionScope, parserSSTResult, argumentNames);
304+
break;
295305
default:
296306
throw new RuntimeException("unexpected mode: " + mode);
297307
}
@@ -323,9 +333,9 @@ private CacheItem parseWithANTLR(ParserMode mode, ParserErrorCallback errors, Py
323333
}
324334

325335
@TruffleBoundary
326-
public Node parseN(ParserMode mode, ParserErrorCallback errors, Source source, Frame currentFrame) {
336+
public Node parseN(ParserMode mode, ParserErrorCallback errors, Source source, Frame currentFrame, String[] argumentNames) {
327337
PythonSSTNodeFactory sstFactory = new PythonSSTNodeFactory(errors, source);
328-
CacheItem parserSSTResult = parseWithANTLR(mode, errors, sstFactory, source, currentFrame);
338+
CacheItem parserSSTResult = parseWithANTLR(mode, errors, sstFactory, source, currentFrame, argumentNames);
329339
try {
330340
return sstFactory.createParserResult(parserSSTResult.antlrResult, mode, currentFrame);
331341
} catch (Exception e) {

0 commit comments

Comments
 (0)