Skip to content

Commit 477cac7

Browse files
msimacekmsimacek
committed
Implement parsing with arguments in bytecode interpreter
1 parent db4fd0e commit 477cac7

File tree

5 files changed

+86
-21
lines changed

5 files changed

+86
-21
lines changed

‎graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/tokenizer/SourceRange.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
package com.oracle.graal.python.pegparser.tokenizer;
4343

4444
public final class SourceRange {
45+
public static final SourceRange ARTIFICIAL_RANGE = new SourceRange(0, 0, 1, 0, 1, 0);
4546

4647
public final int startOffset;
4748
public final int endOffset;

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParseWithArgumentsTests.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
public class ParseWithArgumentsTests extends ParserTestBase {
5454

5555
@Test
56-
public void testSimpple01() throws Exception {
56+
public void testSimple01() throws Exception {
5757
Source source = createSource("arg1");
5858
CallTarget target = context.getEnv().parsePublic(source, "arg1");
5959
assertEquals(66, target.call(66));
@@ -62,7 +62,7 @@ public void testSimpple01() throws Exception {
6262
}
6363

6464
@Test
65-
public void testSimpple02() throws Exception {
65+
public void testSimple02() throws Exception {
6666
Source source = createSource("arg1 + arg2");
6767
CallTarget target = context.getEnv().parsePublic(source, "arg1", "arg2");
6868
assertEquals(11, target.call(5, 6));
@@ -94,7 +94,7 @@ public void testWitouthReturn() throws Exception {
9494
}
9595

9696
@Test
97-
public void testCompareWithAndWithouthArguments() throws Exception {
97+
public void testCompareWithAndWithoutArguments() throws Exception {
9898
Source source = createSource("22");
9999
CallTarget targetWithout = context.getEnv().parsePublic(source);
100100
CallTarget targetWith = context.getEnv().parsePublic(source, "arg1");

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

Lines changed: 79 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.io.IOException;
3535
import java.util.Arrays;
3636
import java.util.EnumSet;
37+
import java.util.List;
3738
import java.util.concurrent.ConcurrentHashMap;
3839
import java.util.concurrent.Semaphore;
3940
import java.util.logging.Level;
@@ -65,13 +66,20 @@
6566
import com.oracle.graal.python.nodes.PRootNode;
6667
import com.oracle.graal.python.nodes.RootNodeFactory;
6768
import com.oracle.graal.python.nodes.bytecode.PBytecodeRootNode;
69+
import com.oracle.graal.python.nodes.call.CallNode;
6870
import com.oracle.graal.python.nodes.control.TopLevelExceptionHandler;
6971
import com.oracle.graal.python.nodes.expression.ExpressionNode;
7072
import com.oracle.graal.python.nodes.util.BadOPCodeNode;
7173
import com.oracle.graal.python.parser.PythonParserImpl;
7274
import com.oracle.graal.python.pegparser.InputType;
75+
import com.oracle.graal.python.pegparser.NodeFactory;
76+
import com.oracle.graal.python.pegparser.NodeFactoryImp;
7377
import com.oracle.graal.python.pegparser.Parser;
78+
import com.oracle.graal.python.pegparser.sst.ArgTy;
79+
import com.oracle.graal.python.pegparser.sst.ArgumentsTy;
7480
import com.oracle.graal.python.pegparser.sst.ModTy;
81+
import com.oracle.graal.python.pegparser.sst.StmtTy;
82+
import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
7583
import com.oracle.graal.python.runtime.GilNode;
7684
import com.oracle.graal.python.runtime.PythonContext;
7785
import com.oracle.graal.python.runtime.PythonContext.PythonThreadState;
@@ -115,6 +123,7 @@
115123
import com.oracle.truffle.api.object.Shape;
116124
import com.oracle.truffle.api.source.Source;
117125
import com.oracle.truffle.api.source.Source.SourceBuilder;
126+
import com.oracle.truffle.api.source.SourceSection;
118127
import com.oracle.truffle.api.strings.TruffleString;
119128

120129
@TruffleLanguage.Registration(id = PythonLanguage.ID, //
@@ -445,10 +454,10 @@ protected CallTarget parse(ParsingRequest request) {
445454
}
446455
}
447456
if (MIME_TYPE_SOURCE_FOR_BYTECODE.equals(source.getMimeType())) {
448-
return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false);
457+
return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false, null);
449458
}
450459
if (MIME_TYPE_SOURCE_FOR_BYTECODE_COMPILE.equals(source.getMimeType())) {
451-
return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false);
460+
return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false, null);
452461
}
453462
throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
454463
}
@@ -457,11 +466,11 @@ private CallTarget parseForBytecodeInterpreter(ParsingRequest request) {
457466
PythonContext context = PythonContext.get(null);
458467
Source source = request.getSource();
459468
if (source.getMimeType() == null || MIME_TYPE.equals(source.getMimeType())) {
460-
if (!request.getArgumentNames().isEmpty()) {
461-
throw new IllegalStateException("Not supported yet");
469+
if (!request.getArgumentNames().isEmpty() && source.isInteractive()) {
470+
throw new IllegalStateException("parse with arguments not allowed for interactive sources");
462471
}
463472
InputType inputType = source.isInteractive() ? InputType.SINGLE : InputType.FILE;
464-
return parseForBytecodeInterpreter(context, source, inputType, true, 0, source.isInteractive());
473+
return parseForBytecodeInterpreter(context, source, inputType, true, 0, source.isInteractive(), request.getArgumentNames());
465474
}
466475
if (!request.getArgumentNames().isEmpty()) {
467476
throw new IllegalStateException("parse with arguments is only allowed for " + MIME_TYPE + " mime type");
@@ -493,46 +502,52 @@ private CallTarget parseForBytecodeInterpreter(ParsingRequest request) {
493502
for (int optimize = 0; optimize < MIME_TYPE_EVAL.length; optimize++) {
494503
if (MIME_TYPE_EVAL[optimize].equals(source.getMimeType())) {
495504
assert !source.isInteractive();
496-
return parseForBytecodeInterpreter(context, source, InputType.EVAL, false, optimize, false);
505+
return parseForBytecodeInterpreter(context, source, InputType.EVAL, false, optimize, false, null);
497506
}
498507
}
499508
for (int optimize = 0; optimize < MIME_TYPE_COMPILE.length; optimize++) {
500509
if (MIME_TYPE_COMPILE[optimize].equals(source.getMimeType())) {
501510
assert !source.isInteractive();
502-
return parseForBytecodeInterpreter(context, source, InputType.FILE, false, optimize, false);
511+
return parseForBytecodeInterpreter(context, source, InputType.FILE, false, optimize, false, null);
503512
}
504513
}
505514
if (MIME_TYPE_SOURCE_FOR_BYTECODE.equals(source.getMimeType())) {
506-
return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false);
515+
return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false, null);
507516
}
508517
if (MIME_TYPE_SOURCE_FOR_BYTECODE_COMPILE.equals(source.getMimeType())) {
509-
return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false);
518+
return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false, null);
510519
}
511520
throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
512521
}
513522

514-
public RootCallTarget parseForBytecodeInterpreter(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal) {
523+
public RootCallTarget parseForBytecodeInterpreter(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List<String> argumentNames) {
515524
RaisePythonExceptionErrorCallback errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(this));
516525
try {
517526
Parser parser = Compiler.createParser(source.getCharacters().toString(), errorCb, type, interactiveTerminal);
518527
Compiler compiler = new Compiler(errorCb);
519528
ModTy mod = (ModTy) parser.parse();
520529
assert mod != null;
530+
boolean hasArguments = argumentNames != null && !argumentNames.isEmpty();
531+
if (hasArguments) {
532+
mod = transformASTForExecutionWithArguments(argumentNames, mod);
533+
}
521534
CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize);
522535
CodeUnit co = cu.assemble();
523-
PBytecodeRootNode bytecodeRootNode = new PBytecodeRootNode(this, co, source, errorCb);
536+
RootNode rootNode = new PBytecodeRootNode(this, co, source, errorCb);
524537
GilNode gil = GilNode.getUncached();
525-
boolean wasAcquired = gil.acquire(context, bytecodeRootNode);
538+
boolean wasAcquired = gil.acquire(context, rootNode);
526539
if (topLevel) {
527540
try {
528541
errorCb.triggerDeprecationWarnings();
529542
} finally {
530543
gil.release(context, wasAcquired);
531544
}
532545
}
533-
RootNode rootNode = bytecodeRootNode;
546+
if (hasArguments) {
547+
rootNode = new RootNodeWithArguments(this, rootNode);
548+
}
534549
if (topLevel && context.isCoreInitialized()) {
535-
rootNode = new TopLevelExceptionHandler(this, bytecodeRootNode, source);
550+
rootNode = new TopLevelExceptionHandler(this, rootNode, source);
536551
}
537552
return PythonUtils.getOrCreateCallTarget(rootNode);
538553
} catch (PException e) {
@@ -543,6 +558,56 @@ public RootCallTarget parseForBytecodeInterpreter(PythonContext context, Source
543558
}
544559
}
545560

561+
private ModTy transformASTForExecutionWithArguments(List<String> argumentNames, ModTy mod) {
562+
NodeFactory nodeFactory = new NodeFactoryImp();
563+
ArgTy[] astArgArray = new ArgTy[argumentNames.size()];
564+
for (int i = 0; i < astArgArray.length; i++) {
565+
astArgArray[i] = nodeFactory.createArgument(argumentNames.get(i), null, null, SourceRange.ARTIFICIAL_RANGE);
566+
}
567+
ArgumentsTy astArgs = nodeFactory.createArguments(null, null, astArgArray, null, null);
568+
StmtTy[] body = ((ModTy.Module) mod).body;
569+
if (body != null && body.length > 0) {
570+
body = Arrays.copyOf(body, body.length);
571+
StmtTy lastStmt = body[body.length - 1];
572+
if (lastStmt instanceof StmtTy.Expr) {
573+
body[body.length - 1] = nodeFactory.createReturn(((StmtTy.Expr) lastStmt).value, lastStmt.getSourceRange());
574+
}
575+
}
576+
String fnName = "execute";
577+
StmtTy astFunction = nodeFactory.createFunctionDef(fnName, astArgs, body, null, null, SourceRange.ARTIFICIAL_RANGE);
578+
/*
579+
* We cannot use a return in a module, but we piggy-back on the fact that we return the last
580+
* expression in a module (see Compiler)
581+
*/
582+
StmtTy astGetFunction = nodeFactory.createExpression(nodeFactory.createVariable(fnName, SourceRange.ARTIFICIAL_RANGE));
583+
return nodeFactory.createModule(new StmtTy[]{astFunction, astGetFunction}, SourceRange.ARTIFICIAL_RANGE);
584+
}
585+
586+
private static class RootNodeWithArguments extends RootNode {
587+
private final RootNode innerRootNode;
588+
@Child private DirectCallNode callModuleNode;
589+
@Child private CallNode callFunctionNode;
590+
591+
public RootNodeWithArguments(PythonLanguage language, RootNode innerRootNode) {
592+
super(language);
593+
this.innerRootNode = innerRootNode;
594+
callModuleNode = DirectCallNode.create(innerRootNode.getCallTarget());
595+
callFunctionNode = CallNode.create();
596+
}
597+
598+
@Override
599+
public Object execute(VirtualFrame frame) {
600+
Object function = callModuleNode.call(frame.getArguments());
601+
Object[] arguments = PythonUtils.arrayCopyOfRange(frame.getArguments(), PArguments.USER_ARGUMENTS_OFFSET, frame.getArguments().length);
602+
return callFunctionNode.execute(frame, function, arguments);
603+
}
604+
605+
@Override
606+
public SourceSection getSourceSection() {
607+
return innerRootNode.getSourceSection();
608+
}
609+
}
610+
546611
private RootNode doParse(PythonContext context, Source source, int optimize) {
547612
ParserMode mode;
548613
if (source.isInteractive()) {
@@ -991,5 +1056,4 @@ public RootCallTarget getDescriptorCallTarget(BuiltinMethodDescriptor descriptor
9911056
assert callTarget != null : "Missing call target for builtin slot descriptor " + descriptor;
9921057
return callTarget;
9931058
}
994-
9951059
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@
6060
import java.util.regex.Matcher;
6161
import java.util.regex.Pattern;
6262

63-
import com.oracle.graal.python.builtins.modules.GraalHPyDebugModuleBuiltins;
6463
import org.graalvm.nativeimage.ImageInfo;
6564

6665
import com.oracle.graal.python.PythonLanguage;
@@ -80,6 +79,7 @@
8079
import com.oracle.graal.python.builtins.modules.FcntlModuleBuiltins;
8180
import com.oracle.graal.python.builtins.modules.FunctoolsModuleBuiltins;
8281
import com.oracle.graal.python.builtins.modules.GcModuleBuiltins;
82+
import com.oracle.graal.python.builtins.modules.GraalHPyDebugModuleBuiltins;
8383
import com.oracle.graal.python.builtins.modules.GraalHPyUniversalModuleBuiltins;
8484
import com.oracle.graal.python.builtins.modules.GraalPythonModuleBuiltins;
8585
import com.oracle.graal.python.builtins.modules.ImpModuleBuiltins;
@@ -1152,7 +1152,7 @@ private void loadFile(TruffleString s, TruffleString prefix, PythonModule mod) {
11521152
Supplier<CallTarget> getCode = () -> {
11531153
Source source = getInternalSource(s, prefix);
11541154
if (getContext().getOption(PythonOptions.EnableBytecodeInterpreter)) {
1155-
return getLanguage().parseForBytecodeInterpreter(getContext(), source, InputType.FILE, false, 0, false);
1155+
return getLanguage().parseForBytecodeInterpreter(getContext(), source, InputType.FILE, false, 0, false, null);
11561156
}
11571157
return PythonUtils.getOrCreateCallTarget((RootNode) getParser().parse(ParserMode.File, 0, this, source, null, null));
11581158
};

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1092,7 +1092,7 @@ PCode compile(TruffleString expression, TruffleString filename, TruffleString mo
10921092
} else {
10931093
Source source = PythonLanguage.newSource(context, finalCode, filename, mayBeFromFile, PythonLanguage.MIME_TYPE);
10941094
if (context.getOption(PythonOptions.EnableBytecodeInterpreter)) {
1095-
return context.getLanguage().parseForBytecodeInterpreter(context, source, InputType.SINGLE, false, kwOptimize, false);
1095+
return context.getLanguage().parseForBytecodeInterpreter(context, source, InputType.SINGLE, false, kwOptimize, false, null);
10961096
}
10971097
return PythonUtils.getOrCreateCallTarget((RootNode) getCore().getParser().parse(pm, kwOptimize, getCore(), source, null, null));
10981098
}

0 commit comments

Comments
 (0)