[GR-18396] Don't create parse tree in ANTLR parser.

PullRequest: graalpython/665

Author: lukasstadler

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonParserImpl.java

@@ -71,6 +71,7 @@ private static Python3NewParser getPython3NewParser(Source source, ParserErrorCa
         lexer.removeErrorListeners();
         lexer.addErrorListener(Builder.ERROR_LISTENER);
         Python3NewParser parser = new Python3NewParser(new CommonTokenStream(lexer));
+        parser.setBuildParseTree(false);
         parser.factory = new PythonNodeFactory(errors.getLanguage(), source);
         parser.removeErrorListeners();
         parser.addErrorListener(Builder.ERROR_LISTENER);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/BuilderNew.java

@@ -57,6 +57,7 @@ public static Python3NewParser createParser(CharStream input) {
         lexer.removeErrorListeners();
         lexer.addErrorListener(Builder.ERROR_LISTENER);
         Python3NewParser parser = new Python3NewParser(new CommonTokenStream(lexer));
+        parser.setBuildParseTree(false);
         parser.removeErrorListeners();
         parser.addErrorListener(Builder.ERROR_LISTENER);
         return parser;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3New.g4

@@ -269,7 +269,7 @@ import java.util.Arrays;
             // We don't have to have new lines in the source section
             int tokenIndex = token.getTokenIndex();
             Token tmp = token;
-            while(tmp.getType() == NEWLINE && tokenIndex > -1) {
+            while(tmp.getType() == NEWLINE && tokenIndex > 0) {
                 tmp = getTokenStream().get(--tokenIndex);
             }
             stopIndex = tmp.getStopIndex();
@@ -280,17 +280,8 @@ import java.util.Arrays;
 
     /** Get the last offset of the context */
     private int getLastIndex(ParserRuleContext ctx) {
-        if (ctx.getStop() != null) {
-            return getStopIndex(ctx);
-        }
-        ParseTree pt = ctx.getChild(ctx.getChildCount() - 1);
-        if (pt instanceof TerminalNode) {
-	    return getStopIndex(((TerminalNode) pt).getSymbol());
-        }
-        if (pt instanceof RuleNode) {
-            return getStopIndex((RuleNode) pt);
-        }
-        return -1;
+    	// ignores ctx
+        return getStopIndex(this._input.get(this._input.index() - 1));
     }
 }
 
@@ -385,12 +376,12 @@ decorated:
 async_funcdef: ASYNC funcdef;
 funcdef
 :
-	'def' NAME parameters
+	'def' n=NAME parameters
 	(
 		'->' test
 	)? ':' 
 	{ 
-            String name = _localctx.NAME().getText(); 
+            String name = $n.getText(); 
             ScopeInfo enclosingScope = factory.getCurrentScope();
             String enclosingClassName = enclosingScope.isInClassScope() ? enclosingScope.getScopeId() : null;
             ScopeInfo functionScope = factory.createScope(name, ScopeInfo.ScopeKind.Function); 
@@ -1395,10 +1386,11 @@ setlisttuplemaker [PythonBuiltinClassType type, PythonBuiltinClassType compType]
 				star_expr { push($star_expr.result); }
 			)
 		)*
-		','?
+		{ boolean comma = false; }
+		(',' { comma = true; } )?
 		{ 
                     SSTNode[] items = getArray(start, SSTNode[].class);
-                    if ($type == PythonBuiltinClassType.PTuple && items.length == 1 && !$ctx.getText().endsWith(",")) {
+                    if ($type == PythonBuiltinClassType.PTuple && items.length == 1 && !comma) {
                         $result = items[0];
                     } else {
                         $result = new CollectionSSTNode(items, $type, -1, -1); 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/sst/BlockSSTNode.java

@@ -41,10 +41,6 @@
 
 package com.oracle.graal.python.parser.sst;
 
-/**
- *
- * @author petr
- */
 public class BlockSSTNode extends SSTNode {
     protected final SSTNode[] statements;