spaces before EOF are skipped, so check the upcoming token rather than the upcoming char to finish lexing

Author: timfel

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.g4

@@ -57,8 +57,10 @@ tokens { INDENT, DEDENT }
 
   @Override
   public Token nextToken() {
+    Token next = super.nextToken();
+
     // Check if the end-of-file is ahead to insert any missing DEDENTS and a NEWLINE.
-    if (_input.LA(1) == EOF && !expandedEOF) {
+    if (next.getType() == EOF && !expandedEOF) {
       expandedEOF = true;
 
       // Remove any trailing EOF tokens from our buffer.
@@ -78,10 +80,10 @@ tokens { INDENT, DEDENT }
       }
 
       // Put the EOF back on the token stream.
-      this.emit(commonToken(Python3Parser.EOF, "<EOF>"));
-    }
+      this.emit(next);
 
-    Token next = super.nextToken();
+      next = super.nextToken();
+    }
 
     if (next.getChannel() == Token.DEFAULT_CHANNEL) {
       // Keep track of the last token on the default channel.

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3Lexer.java

@@ -175,8 +175,10 @@ public void emit(Token t) {
 
 	  @Override
 	  public Token nextToken() {
+	    Token next = super.nextToken();
+
 	    // Check if the end-of-file is ahead to insert any missing DEDENTS and a NEWLINE.
-	    if (_input.LA(1) == EOF && !expandedEOF) {
+	    if (next.getType() == EOF && !expandedEOF) {
 	      expandedEOF = true;
 
 	      // Remove any trailing EOF tokens from our buffer.
@@ -196,10 +198,10 @@ public Token nextToken() {
 	      }
 
 	      // Put the EOF back on the token stream.
-	      this.emit(commonToken(Python3Parser.EOF, "<EOF>"));
-	    }
+	      this.emit(next);
 
-	    Token next = super.nextToken();
+	      next = super.nextToken();
+	    }
 
 	    if (next.getChannel() == Token.DEFAULT_CHANNEL) {
 	      // Keep track of the last token on the default channel.