change scope parsing to single-pass-and-fixup (instead of two-pass), make tree translation less stateful, store scopes in parse tree

Author: lukasstadler

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/NodeFactory.java

@@ -27,7 +27,6 @@
 
 import java.math.BigInteger;
 import java.util.List;
-import java.util.Set;
 
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.objects.complex.PComplex;
@@ -274,7 +273,7 @@ public PNode createListLiteral(PNode[] values) {
         return ListLiteralNode.create(values);
     }
 
-    public PNode createSetLiteral(Set<PNode> values) {
+    public PNode createSetLiteral(List<PNode> values) {
         PNode[] convertedValues = values.toArray(new PNode[values.size()]);
         return new SetLiteralNode(convertedValues);
     }
@@ -524,7 +523,7 @@ public PNode createSetAttribute(PNode object, String key, PNode rhs) {
         return SetAttributeNode.create(key, object, rhs);
     }
 
-    public PNode createDestructuringAssignment(PNode rhs, List<ReadNode> slots, int starredIndex, PNode[] assignments) {
+    public PNode createDestructuringAssignment(PNode rhs, ReadNode[] slots, int starredIndex, PNode[] assignments) {
         return DestructuringAssignmentNode.create(rhs, slots, starredIndex, assignments);
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/PNode.java

@@ -45,6 +45,9 @@
 @ImportStatic({PGuards.class, PythonOptions.class, SpecialMethodNames.class, SpecialAttributeNames.class, BuiltinNames.class})
 @GenerateWrapper
 public abstract class PNode extends PBaseNode implements InstrumentableNode {
+
+    public static final PNode[] EMPTY_ARRAY = new PNode[0];
+
     @CompilationFinal private SourceSection sourceSection;
     @CompilationFinal private boolean isStmt = false;
     @CompilationFinal private boolean isRoot = false;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/frame/DestructuringAssignmentNode.java

@@ -32,7 +32,6 @@
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.SyntaxError;
 
 import java.util.Arrays;
-import java.util.List;
 
 import com.oracle.graal.python.builtins.modules.BuiltinFunctions;
 import com.oracle.graal.python.builtins.modules.BuiltinFunctionsFactory;
@@ -68,18 +67,18 @@ public final class DestructuringAssignmentNode extends PNode implements WriteNod
     private final ConditionProfile errorProfile2 = ConditionProfile.createBinaryProfile();
     private final int starredIndex;
 
-    public DestructuringAssignmentNode(PNode rhs, List<ReadNode> slots, int starredIndex, PNode[] assignments) {
+    public DestructuringAssignmentNode(PNode rhs, ReadNode[] slots, int starredIndex, PNode[] assignments) {
         this.rhs = rhs;
         this.starredIndex = starredIndex;
         this.assignments = assignments;
-        this.slots = new WriteNode[slots.size()];
-        for (int i = 0; i < slots.size(); i++) {
-            this.slots[i] = (WriteNode) slots.get(i).makeWriteNode(null);
+        this.slots = new WriteNode[slots.length];
+        for (int i = 0; i < slots.length; i++) {
+            this.slots[i] = (WriteNode) slots[i].makeWriteNode(null);
         }
         this.lenNode = starredIndex == -1 ? null : BuiltinFunctionsFactory.LenNodeFactory.create();
     }
 
-    public static PNode create(PNode rhs, List<ReadNode> slots, int starredIndex, PNode[] assignments) {
+    public static PNode create(PNode rhs, ReadNode[] slots, int starredIndex, PNode[] assignments) {
         return new DestructuringAssignmentNode(rhs, slots, starredIndex, assignments);
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/frame/FrameSlotIDs.java

@@ -27,5 +27,4 @@
 
 public abstract class FrameSlotIDs {
     public static final String RETURN_SLOT_ID = "<return_val>";
-    public static final String LIST_COMPREHENSION_SLOT_ID = "<list_comp_val>";
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/function/BuiltinFunctionRootNode.java

@@ -145,7 +145,7 @@ public Object execute(VirtualFrame frame) {
     }
 
     public BuiltinFunctionRootNode(PythonLanguage language, Builtin builtin, NodeFactory<? extends PythonBuiltinBaseNode> factory, boolean declaresExplicitSelf) {
-        super(language, null);
+        super(language);
         this.builtin = builtin;
         this.factory = factory;
         this.declaresExplicitSelf = declaresExplicitSelf;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/AssignmentTranslator.java

@@ -59,10 +59,10 @@ public class AssignmentTranslator extends Python3BaseVisitor<PNode> {
 
     private final NodeFactory factory;
     private final TranslationEnvironment environment;
-    private final PythonBaseTreeTranslator<?> translator;
+    private final PythonTreeTranslator translator;
     private final PythonCore core;
 
-    public AssignmentTranslator(PythonCore core, TranslationEnvironment environment, PythonBaseTreeTranslator<?> translator) {
+    public AssignmentTranslator(PythonCore core, TranslationEnvironment environment, PythonTreeTranslator translator) {
         this.core = core;
         this.factory = core.getLanguage().getNodeFactory();
         this.environment = environment;
@@ -103,37 +103,37 @@ private PNode createAssignment(PNode lhs, PNode rhs) {
     }
 
     private PNode createDestructuringAssignment(PNode[] leftHandSides, PNode rhs) {
-        List<PNode> statements = new ArrayList<>();
-        List<ReadNode> temps = new ArrayList<>();
+        PNode[] statements = new PNode[leftHandSides.length];
+        ReadNode[] temps = new ReadNode[leftHandSides.length];
         int starredIndex = -1;
         for (int i = 0; i < leftHandSides.length; i++) {
             ReadNode tempRead = environment.makeTempLocalVariable();
-            temps.add(tempRead);
+            temps[i] = tempRead;
             if (leftHandSides[i] instanceof StarredExpressionNode) {
                 if (starredIndex != -1) {
                     throw core.raise(SyntaxError, "two starred expressions in assignment");
                 }
                 starredIndex = i;
-                statements.add(createAssignment(((StarredExpressionNode) leftHandSides[i]).getValue(), (PNode) tempRead));
+                statements[i] = createAssignment(((StarredExpressionNode) leftHandSides[i]).getValue(), (PNode) tempRead);
             } else {
-                statements.add(createAssignment(leftHandSides[i], (PNode) tempRead));
+                statements[i] = createAssignment(leftHandSides[i], (PNode) tempRead);
             }
         }
-        return factory.createDestructuringAssignment(rhs, temps, starredIndex, statements.toArray(new PNode[0]));
+        return factory.createDestructuringAssignment(rhs, temps, starredIndex, statements);
     }
 
     private PNode createMultiAssignment(List<NormassignContext> normassign, PNode mostRhs, PNode mostLhs) {
         ReadNode tmp = environment.makeTempLocalVariable();
         PNode tmpWrite = tmp.makeWriteNode(mostRhs);
-        List<PNode> assignments = new ArrayList<>();
-        assignments.add(tmpWrite);
-        assignments.add(createAssignment(mostLhs, (PNode) tmp));
+        PNode[] assignments = new PNode[normassign.size() + 1];
+        assignments[0] = tmpWrite;
+        assignments[1] = createAssignment(mostLhs, (PNode) tmp);
         for (int i = 0; i < normassign.size() - 1; i++) {
             NormassignContext normassignContext = normassign.get(i);
             if (normassignContext.yield_expr() != null) {
                 throw core.raise(SyntaxError, "assignment to yield expression not possible");
             }
-            assignments.add(createAssignment(normassignContext.accept(this), (PNode) tmp));
+            assignments[i + 2] = createAssignment(normassignContext.accept(this), (PNode) tmp);
         }
         return factory.createBlock(assignments);
     }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/InlineScopeTranslator.java

@@ -27,21 +27,19 @@
 
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.SyntaxError;
 
-import java.util.function.Consumer;
-
 import org.antlr.v4.runtime.CharStreams;
 import org.antlr.v4.runtime.ParserRuleContext;
 
 import com.oracle.graal.python.builtins.objects.exception.PBaseException;
 import com.oracle.graal.python.builtins.objects.type.PythonClass;
-import com.oracle.graal.python.parser.ScopeTranslator.ScopeTranslatorFactory;
 import com.oracle.graal.python.parser.antlr.Builder;
 import com.oracle.graal.python.parser.antlr.Python3Parser;
 import com.oracle.graal.python.runtime.PythonCore;
 import com.oracle.graal.python.runtime.PythonParser;
 import com.oracle.graal.python.runtime.exception.PException;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.frame.Frame;
+import com.oracle.truffle.api.frame.FrameDescriptor;
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.source.Source;
 import com.oracle.truffle.api.source.SourceSection;
@@ -57,6 +55,7 @@ private static Python3Parser getPython3Parser(String string) {
     @Override
     @TruffleBoundary
     public Node parse(ParserMode mode, PythonCore core, Source source, Frame currentFrame) {
+        // ANTLR parsing
         Python3Parser parser = getPython3Parser(source.getCharacters().toString());
         ParserRuleContext input;
         try {
@@ -91,19 +90,19 @@ public Node parse(ParserMode mode, PythonCore core, Source source, Frame current
                 throw handleParserError(core, source, e);
             }
         }
+
+        // prepare scope translator
         TranslationEnvironment environment = new TranslationEnvironment(core.getLanguage());
-        Node result;
-        Consumer<TranslationEnvironment> environmentConsumer = (env) -> env.setFreeVarsInRootScope(currentFrame);
-        if (mode == ParserMode.InlineEvaluation) {
-            ScopeTranslatorFactory scopeTranslator = (env, trackCells) -> new InlineScopeTranslator<>(core, env, currentFrame.getFrameDescriptor(), trackCells);
-            ScopeTranslator.accept(input, environment, scopeTranslator, environmentConsumer);
-            result = new PythonInlineTreeTranslator(core, source.getName(), input, environment, source).getTranslationResult();
-        } else {
-            ScopeTranslatorFactory scopeTranslator = (env, trackCells) -> new ScopeTranslator<>(core, env, source.isInteractive(), trackCells);
-            ScopeTranslator.accept(input, environment, scopeTranslator, environmentConsumer);
-            result = new PythonTreeTranslator(core, source.getName(), input, environment, source).getTranslationResult();
-        }
-        return result;
+        FrameDescriptor inlineLocals = mode == ParserMode.InlineEvaluation ? currentFrame.getFrameDescriptor() : null;
+        ScopeTranslator<Object> defineScopes = new ScopeTranslator<>(core, environment, source.isInteractive(), inlineLocals);
+        // first pass of the scope translator -> define the scopes
+        input.accept(defineScopes);
+        // create frame slots for cell and free vars
+        defineScopes.setFreeVarsInRootScope(currentFrame);
+        defineScopes.createFrameSlotsForCellAndFreeVars();
+
+        // create Truffle ASTs
+        return PythonTreeTranslator.translate(core, source.getName(), input, environment, source, mode == ParserMode.InlineEvaluation);
     }
 
     @Override