[GR-40203] Implement parsing with arguments in bytecode interpreter

msimacek · msimacek · commit 19e2857eadf3 · 2022-08-03T11:12:20.000Z
PullRequest: graalpython/2379
diff --git a/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/tokenizer/SourceRange.java b/graalpython/com.oracle.graal.python.pegparser/src/com/oracle/graal/python/pegparser/tokenizer/SourceRange.java
@@ -42,6 +42,7 @@
 package com.oracle.graal.python.pegparser.tokenizer;
 
 public final class SourceRange {
+    public static final SourceRange ARTIFICIAL_RANGE = new SourceRange(0, 0, 1, 0, 1, 0);
 
     public final int startOffset;
     public final int endOffset;
diff --git a/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParseWithArgumentsTests.java b/graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/parser/ParseWithArgumentsTests.java
@@ -53,7 +53,7 @@
 public class ParseWithArgumentsTests extends ParserTestBase {
 
     @Test
-    public void testSimpple01() throws Exception {
+    public void testSimple01() throws Exception {
         Source source = createSource("arg1");
         CallTarget target = context.getEnv().parsePublic(source, "arg1");
         assertEquals(66, target.call(66));
@@ -62,7 +62,7 @@ public void testSimpple01() throws Exception {
     }
 
     @Test
-    public void testSimpple02() throws Exception {
+    public void testSimple02() throws Exception {
         Source source = createSource("arg1 + arg2");
         CallTarget target = context.getEnv().parsePublic(source, "arg1", "arg2");
         assertEquals(11, target.call(5, 6));
@@ -94,7 +94,7 @@ public void testWitouthReturn() throws Exception {
     }
 
     @Test
-    public void testCompareWithAndWithouthArguments() throws Exception {
+    public void testCompareWithAndWithoutArguments() throws Exception {
         Source source = createSource("22");
         CallTarget targetWithout = context.getEnv().parsePublic(source);
         CallTarget targetWith = context.getEnv().parsePublic(source, "arg1");
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testImportAs.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testImportAs.co
@@ -7,4 +7,6 @@ Disassembly of <module>:
 000000         9 POP_TOP
 000000        10 IMPORT_FROM                       2    (c)
 000000        12 STORE_NAME                        3    (d)
-000000        14 RETURN_VALUE
+000000        14 POP_TOP
+000000        15 LOAD_NONE
+000000        16 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testImportFrom.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testImportFrom.co
@@ -6,4 +6,6 @@ Disassembly of <module>:
 000000         8 STORE_NAME                        1    (sqrt)
 000000        10 IMPORT_FROM                       2    (sin)
 000000        12 STORE_NAME                        3    (sine)
-000000        14 RETURN_VALUE
+000000        14 POP_TOP
+000000        15 LOAD_NONE
+000000        16 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testLoadClassDefRef.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testLoadClassDefRef.co
@@ -4,7 +4,7 @@ Disassembly of <module>:
 000030         5 LOAD_NAME                         0    (f)
 000032         7 LOAD_BYTE_O                       1
 000030         9 CALL_FUNCTION                     1
-000030        11 RETURN_VALUE
+000000        11 RETURN_VALUE
 
 Disassembly of f:
 000015         0 LOAD_BUILD_CLASS
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExcept.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExcept.co
@@ -47,4 +47,4 @@ Disassembly of <module>:
 000121   >>   78 LOAD_NAME                         0    (print)
 000127        80 LOAD_STRING                       4    ('after')
 000121        82 CALL_FUNCTION                     1
-000121        84 RETURN_VALUE
+000104        84 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptBare.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptBare.co
@@ -37,4 +37,4 @@ Disassembly of <module>:
 000109   >>   59 LOAD_NAME                         0    (print)
 000115        61 LOAD_STRING                       4    ('after')
 000109        63 CALL_FUNCTION                     1
-000109        65 RETURN_VALUE
+000088        65 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptElse.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptElse.co
@@ -52,4 +52,4 @@ Disassembly of <module>:
 000143   >>   87 LOAD_NAME                         0    (print)
 000149        89 LOAD_STRING                       5    ('after')
 000143        91 CALL_FUNCTION                     1
-000143        93 RETURN_VALUE
+000129        93 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptElseFinally.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptElseFinally.co
@@ -68,4 +68,4 @@ Disassembly of <module>:
 000171       112 LOAD_NAME                         0    (print)
 000177       114 LOAD_STRING                       6    ('after')
 000171       116 CALL_FUNCTION                     1
-000171       118 RETURN_VALUE
+000154       118 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptFinally.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryExceptFinally.co
@@ -63,4 +63,4 @@ Disassembly of <module>:
 000149       103 LOAD_NAME                         0    (print)
 000155       105 LOAD_STRING                       5    ('after')
 000149       107 CALL_FUNCTION                     1
-000149       109 RETURN_VALUE
+000132       109 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryFinally.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testTryFinally.co
@@ -21,4 +21,4 @@ Disassembly of <module>:
 000064        32 LOAD_NAME                         0    (print)
 000070        34 LOAD_STRING                       3    ('after')
 000064        36 CALL_FUNCTION                     1
-000064        38 RETURN_VALUE
+000047        38 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testWith.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testWith.co
@@ -18,4 +18,4 @@ Disassembly of <module>:
 000062        27 LOAD_NAME                         0    (print)
 000068        29 LOAD_STRING                       3    ('after')
 000062        31 CALL_FUNCTION                     1
-000062        33 RETURN_VALUE
+000016        33 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testWithMultiple.co b/graalpython/com.oracle.graal.python.test/testData/goldenFiles/CompilerTests/testWithMultiple.co
@@ -25,4 +25,4 @@ Disassembly of <module>:
 000080        37 LOAD_NAME                         0    (print)
 000086        39 LOAD_STRING                       4    ('after')
 000080        41 CALL_FUNCTION                     1
-000080        43 RETURN_VALUE
+000016        43 RETURN_VALUE
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java
@@ -34,6 +34,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.EnumSet;
+import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Semaphore;
 import java.util.logging.Level;
@@ -65,13 +66,20 @@
 import com.oracle.graal.python.nodes.PRootNode;
 import com.oracle.graal.python.nodes.RootNodeFactory;
 import com.oracle.graal.python.nodes.bytecode.PBytecodeRootNode;
+import com.oracle.graal.python.nodes.call.CallNode;
 import com.oracle.graal.python.nodes.control.TopLevelExceptionHandler;
 import com.oracle.graal.python.nodes.expression.ExpressionNode;
 import com.oracle.graal.python.nodes.util.BadOPCodeNode;
 import com.oracle.graal.python.parser.PythonParserImpl;
 import com.oracle.graal.python.pegparser.InputType;
+import com.oracle.graal.python.pegparser.NodeFactory;
+import com.oracle.graal.python.pegparser.NodeFactoryImp;
 import com.oracle.graal.python.pegparser.Parser;
+import com.oracle.graal.python.pegparser.sst.ArgTy;
+import com.oracle.graal.python.pegparser.sst.ArgumentsTy;
 import com.oracle.graal.python.pegparser.sst.ModTy;
+import com.oracle.graal.python.pegparser.sst.StmtTy;
+import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
 import com.oracle.graal.python.runtime.GilNode;
 import com.oracle.graal.python.runtime.PythonContext;
 import com.oracle.graal.python.runtime.PythonContext.PythonThreadState;
@@ -115,6 +123,7 @@
 import com.oracle.truffle.api.object.Shape;
 import com.oracle.truffle.api.source.Source;
 import com.oracle.truffle.api.source.Source.SourceBuilder;
+import com.oracle.truffle.api.source.SourceSection;
 import com.oracle.truffle.api.strings.TruffleString;
 
 @TruffleLanguage.Registration(id = PythonLanguage.ID, //
@@ -445,10 +454,10 @@ protected CallTarget parse(ParsingRequest request) {
             }
         }
         if (MIME_TYPE_SOURCE_FOR_BYTECODE.equals(source.getMimeType())) {
-            return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false);
+            return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false, null);
         }
         if (MIME_TYPE_SOURCE_FOR_BYTECODE_COMPILE.equals(source.getMimeType())) {
-            return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false);
+            return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false, null);
         }
         throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
     }
@@ -457,11 +466,11 @@ private CallTarget parseForBytecodeInterpreter(ParsingRequest request) {
         PythonContext context = PythonContext.get(null);
         Source source = request.getSource();
         if (source.getMimeType() == null || MIME_TYPE.equals(source.getMimeType())) {
-            if (!request.getArgumentNames().isEmpty()) {
-                throw new IllegalStateException("Not supported yet");
+            if (!request.getArgumentNames().isEmpty() && source.isInteractive()) {
+                throw new IllegalStateException("parse with arguments not allowed for interactive sources");
             }
             InputType inputType = source.isInteractive() ? InputType.SINGLE : InputType.FILE;
-            return parseForBytecodeInterpreter(context, source, inputType, true, 0, source.isInteractive());
+            return parseForBytecodeInterpreter(context, source, inputType, true, 0, source.isInteractive(), request.getArgumentNames());
         }
         if (!request.getArgumentNames().isEmpty()) {
             throw new IllegalStateException("parse with arguments is only allowed for " + MIME_TYPE + " mime type");
@@ -493,46 +502,52 @@ private CallTarget parseForBytecodeInterpreter(ParsingRequest request) {
         for (int optimize = 0; optimize < MIME_TYPE_EVAL.length; optimize++) {
             if (MIME_TYPE_EVAL[optimize].equals(source.getMimeType())) {
                 assert !source.isInteractive();
-                return parseForBytecodeInterpreter(context, source, InputType.EVAL, false, optimize, false);
+                return parseForBytecodeInterpreter(context, source, InputType.EVAL, false, optimize, false, null);
             }
         }
         for (int optimize = 0; optimize < MIME_TYPE_COMPILE.length; optimize++) {
             if (MIME_TYPE_COMPILE[optimize].equals(source.getMimeType())) {
                 assert !source.isInteractive();
-                return parseForBytecodeInterpreter(context, source, InputType.FILE, false, optimize, false);
+                return parseForBytecodeInterpreter(context, source, InputType.FILE, false, optimize, false, null);
             }
         }
         if (MIME_TYPE_SOURCE_FOR_BYTECODE.equals(source.getMimeType())) {
-            return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false);
+            return parseForBytecodeInterpreter(context, source, InputType.FILE, true, 0, false, null);
         }
         if (MIME_TYPE_SOURCE_FOR_BYTECODE_COMPILE.equals(source.getMimeType())) {
-            return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false);
+            return parseForBytecodeInterpreter(context, source, InputType.FILE, false, 0, false, null);
         }
         throw CompilerDirectives.shouldNotReachHere("unknown mime type: " + source.getMimeType());
     }
 
-    public RootCallTarget parseForBytecodeInterpreter(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal) {
+    public RootCallTarget parseForBytecodeInterpreter(PythonContext context, Source source, InputType type, boolean topLevel, int optimize, boolean interactiveTerminal, List<String> argumentNames) {
         RaisePythonExceptionErrorCallback errorCb = new RaisePythonExceptionErrorCallback(source, PythonOptions.isPExceptionWithJavaStacktrace(this));
         try {
             Parser parser = Compiler.createParser(source.getCharacters().toString(), errorCb, type, interactiveTerminal);
             Compiler compiler = new Compiler(errorCb);
             ModTy mod = (ModTy) parser.parse();
             assert mod != null;
+            boolean hasArguments = argumentNames != null && !argumentNames.isEmpty();
+            if (hasArguments) {
+                mod = transformASTForExecutionWithArguments(argumentNames, mod);
+            }
             CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize);
             CodeUnit co = cu.assemble();
-            PBytecodeRootNode bytecodeRootNode = new PBytecodeRootNode(this, co, source, errorCb);
+            RootNode rootNode = new PBytecodeRootNode(this, co, source, errorCb);
             GilNode gil = GilNode.getUncached();
-            boolean wasAcquired = gil.acquire(context, bytecodeRootNode);
+            boolean wasAcquired = gil.acquire(context, rootNode);
             if (topLevel) {
                 try {
                     errorCb.triggerDeprecationWarnings();
                 } finally {
                     gil.release(context, wasAcquired);
                 }
             }
-            RootNode rootNode = bytecodeRootNode;
+            if (hasArguments) {
+                rootNode = new RootNodeWithArguments(this, rootNode);
+            }
             if (topLevel && context.isCoreInitialized()) {
-                rootNode = new TopLevelExceptionHandler(this, bytecodeRootNode, source);
+                rootNode = new TopLevelExceptionHandler(this, rootNode, source);
             }
             return PythonUtils.getOrCreateCallTarget(rootNode);
         } catch (PException e) {
@@ -543,6 +558,56 @@ public RootCallTarget parseForBytecodeInterpreter(PythonContext context, Source
         }
     }
 
+    private ModTy transformASTForExecutionWithArguments(List<String> argumentNames, ModTy mod) {
+        NodeFactory nodeFactory = new NodeFactoryImp();
+        ArgTy[] astArgArray = new ArgTy[argumentNames.size()];
+        for (int i = 0; i < astArgArray.length; i++) {
+            astArgArray[i] = nodeFactory.createArgument(argumentNames.get(i), null, null, SourceRange.ARTIFICIAL_RANGE);
+        }
+        ArgumentsTy astArgs = nodeFactory.createArguments(null, null, astArgArray, null, null);
+        StmtTy[] body = ((ModTy.Module) mod).body;
+        if (body != null && body.length > 0) {
+            body = Arrays.copyOf(body, body.length);
+            StmtTy lastStmt = body[body.length - 1];
+            if (lastStmt instanceof StmtTy.Expr) {
+                body[body.length - 1] = nodeFactory.createReturn(((StmtTy.Expr) lastStmt).value, lastStmt.getSourceRange());
+            }
+        }
+        String fnName = "execute";
+        StmtTy astFunction = nodeFactory.createFunctionDef(fnName, astArgs, body, null, null, SourceRange.ARTIFICIAL_RANGE);
+        /*
+         * We cannot use a return in a module, but we piggy-back on the fact that we return the last
+         * expression in a module (see Compiler)
+         */
+        StmtTy astGetFunction = nodeFactory.createExpression(nodeFactory.createVariable(fnName, SourceRange.ARTIFICIAL_RANGE));
+        return nodeFactory.createModule(new StmtTy[]{astFunction, astGetFunction}, SourceRange.ARTIFICIAL_RANGE);
+    }
+
+    private static class RootNodeWithArguments extends RootNode {
+        private final RootNode innerRootNode;
+        @Child private DirectCallNode callModuleNode;
+        @Child private CallNode callFunctionNode;
+
+        public RootNodeWithArguments(PythonLanguage language, RootNode innerRootNode) {
+            super(language);
+            this.innerRootNode = innerRootNode;
+            callModuleNode = DirectCallNode.create(innerRootNode.getCallTarget());
+            callFunctionNode = CallNode.create();
+        }
+
+        @Override
+        public Object execute(VirtualFrame frame) {
+            Object function = callModuleNode.call(frame.getArguments());
+            Object[] arguments = PythonUtils.arrayCopyOfRange(frame.getArguments(), PArguments.USER_ARGUMENTS_OFFSET, frame.getArguments().length);
+            return callFunctionNode.execute(frame, function, arguments);
+        }
+
+        @Override
+        public SourceSection getSourceSection() {
+            return innerRootNode.getSourceSection();
+        }
+    }
+
     private RootNode doParse(PythonContext context, Source source, int optimize) {
         ParserMode mode;
         if (source.isInteractive()) {
@@ -991,5 +1056,4 @@ public RootCallTarget getDescriptorCallTarget(BuiltinMethodDescriptor descriptor
         assert callTarget != null : "Missing call target for builtin slot descriptor " + descriptor;
         return callTarget;
     }
-
 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java
@@ -60,7 +60,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import com.oracle.graal.python.builtins.modules.GraalHPyDebugModuleBuiltins;
 import org.graalvm.nativeimage.ImageInfo;
 
 import com.oracle.graal.python.PythonLanguage;
@@ -80,6 +79,7 @@
 import com.oracle.graal.python.builtins.modules.FcntlModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.FunctoolsModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.GcModuleBuiltins;
+import com.oracle.graal.python.builtins.modules.GraalHPyDebugModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.GraalHPyUniversalModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.GraalPythonModuleBuiltins;
 import com.oracle.graal.python.builtins.modules.ImpModuleBuiltins;
@@ -1152,7 +1152,7 @@ private void loadFile(TruffleString s, TruffleString prefix, PythonModule mod) {
         Supplier<CallTarget> getCode = () -> {
             Source source = getInternalSource(s, prefix);
             if (getContext().getOption(PythonOptions.EnableBytecodeInterpreter)) {
-                return getLanguage().parseForBytecodeInterpreter(getContext(), source, InputType.FILE, false, 0, false);
+                return getLanguage().parseForBytecodeInterpreter(getContext(), source, InputType.FILE, false, 0, false, null);
             }
             return PythonUtils.getOrCreateCallTarget((RootNode) getParser().parse(ParserMode.File, 0, this, source, null, null));
         };
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java
@@ -1092,7 +1092,7 @@ PCode compile(TruffleString expression, TruffleString filename, TruffleString mo
                 } else {
                     Source source = PythonLanguage.newSource(context, finalCode, filename, mayBeFromFile, PythonLanguage.MIME_TYPE);
                     if (context.getOption(PythonOptions.EnableBytecodeInterpreter)) {
-                        return context.getLanguage().parseForBytecodeInterpreter(context, source, InputType.SINGLE, false, kwOptimize, false);
+                        return context.getLanguage().parseForBytecodeInterpreter(context, source, InputType.SINGLE, false, kwOptimize, false, null);
                     }
                     return PythonUtils.getOrCreateCallTarget((RootNode) getCore().getParser().parse(pm, kwOptimize, getCore(), source, null, null));
                 }
diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/Compiler.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/compiler/Compiler.java
