[GR-23224] Make test_eof pass.

Author: ppisl

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_eof.txt

@@ -0,0 +1,4 @@
+*graalpython.lib-python.3.test.test_eof.EOFTestCase.test_EOFC
+*graalpython.lib-python.3.test.test_eof.EOFTestCase.test_EOFS
+*graalpython.lib-python.3.test.test_eof.EOFTestCase.test_line_continuation_EOF
+*graalpython.lib-python.3.test.test_eof.EOFTestCase.test_line_continuation_EOF_from_file_bpo2180

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/DescriptiveBailErrorListener.java

@@ -111,6 +111,21 @@ public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
                     entireMessage = "unexpected unindent";
                     errorType = ErrorType.Indentation;
                     break;
+                case Python3Parser.LONG_QUOTES1:
+                case Python3Parser.LONG_QUOTES2:
+                    entireMessage = "EOF while scanning triple-quoted string literal";
+                    break;
+                case Python3Parser.LINE_JOINING_EOF_ERROR:
+                    entireMessage = "unexpected EOF while parsing";
+                    break;
+                case Python3Parser.UNKNOWN_CHAR:
+                    String text = token.getText();
+                    if (text.startsWith("\'") || text.startsWith("\"")) {
+                        entireMessage = "EOL while scanning string literal";
+                    } else if (text.equals("\\")) {
+                        entireMessage = "unexpected character after line continuation character";
+                    }
+                    break;
                 default:
                     if (expectedTokens != null && expectedTokens.contains(Python3Parser.INDENT)) {
                         entireMessage = "expected an indented block";

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.g4

@@ -35,7 +35,10 @@ grammar Python3;
 // All comments that start with "///" are copy-pasted from
 // The Python Language Reference
 
-tokens { INDENT, DEDENT, INDENT_ERROR, TAB_ERROR }
+tokens { INDENT, DEDENT, INDENT_ERROR, TAB_ERROR, 
+        LINE_JOINING_EOF_ERROR // This is error token emitted, there is line continuation just before EOF.
+                                // Line continuation is in the hidden channel, so normally is not emitted.
+    }
 
 @lexer::members {
   // new version with semantic actions in parser
@@ -178,6 +181,15 @@ tokens { INDENT, DEDENT, INDENT_ERROR, TAB_ERROR }
     return dedent;
   }
 
+  private Token createLineContinuationEOFError() {
+    // this has to be called only at the end of the file
+    CommonToken errorToken = commonToken(Python3Parser.LINE_JOINING_EOF_ERROR, "");
+    // set the position at the previous char, which has to be line continuation
+    errorToken.setStartIndex(_input.index() - 1);
+    errorToken.setStopIndex(_input.index() - 1);
+    return errorToken;
+  }
+  
   private Token createIndentError(int type) {
     // For some reason, CPython sets the error position to the end of line
     int cur = getCharIndex();
@@ -1960,13 +1972,29 @@ LONG_QUOTES1 : '"""' {usedQuote1();};
 LONG_QUOTES2 : '\'\'\'' {usedQuote2();};
 
 SKIP_
- : ( SPACES | COMMENT | LINE_JOINING ) -> skip
+ : ( SPACES 
+| COMMENT 
+| LINE_JOINING 
+    {   
+        // We need to hanle line continuation here, because normally is hidden for the parser
+        // but we need to handle the case, where is just before EOF.
+        if (_input.size() == _input.index()) {
+            this.emit(createLineContinuationEOFError());
+        }
+    }
+)  -> skip
  ;
 
 BOM : '\uFEFF';
 
 UNKNOWN_CHAR
- : .
+ : . 
+    {
+        // check the case when the char is line continuation just before EOF
+        if (_input.size() == _input.index() && "\\".equals(_input.getText(Interval.of(_input.size()- 1, _input.size())))) {
+            this.emit(createLineContinuationEOFError());
+        }
+    }
  ;
 
 /* 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.interp

@@ -100,6 +100,7 @@ INDENT=99
 DEDENT=100
 INDENT_ERROR=101
 TAB_ERROR=102
+LINE_JOINING_EOF_ERROR=103
 'def'=2
 'return'=3
 'raise'=4

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.tokens

@@ -86,7 +86,7 @@ public class Python3Parser extends Parser {
 		MOD_ASSIGN=86, AND_ASSIGN=87, OR_ASSIGN=88, XOR_ASSIGN=89, LEFT_SHIFT_ASSIGN=90, 
 		RIGHT_SHIFT_ASSIGN=91, POWER_ASSIGN=92, IDIV_ASSIGN=93, LONG_QUOTES1=94, 
 		LONG_QUOTES2=95, SKIP_=96, BOM=97, UNKNOWN_CHAR=98, INDENT=99, DEDENT=100, 
-		INDENT_ERROR=101, TAB_ERROR=102;
+		INDENT_ERROR=101, TAB_ERROR=102, LINE_JOINING_EOF_ERROR=103;
 	public static final int
 		RULE_single_input = 0, RULE_file_input = 1, RULE_withArguments_input = 2, 
 		RULE_eval_input = 3, RULE_decorator = 4, RULE_decorators = 5, RULE_decorated = 6, 
@@ -167,7 +167,7 @@ private static String[] makeSymbolicNames() {
 			"MOD_ASSIGN", "AND_ASSIGN", "OR_ASSIGN", "XOR_ASSIGN", "LEFT_SHIFT_ASSIGN", 
 			"RIGHT_SHIFT_ASSIGN", "POWER_ASSIGN", "IDIV_ASSIGN", "LONG_QUOTES1", 
 			"LONG_QUOTES2", "SKIP_", "BOM", "UNKNOWN_CHAR", "INDENT", "DEDENT", "INDENT_ERROR", 
-			"TAB_ERROR"
+			"TAB_ERROR", "LINE_JOINING_EOF_ERROR"
 		};
 	}
 	private static final String[] _SYMBOLIC_NAMES = makeSymbolicNames();
@@ -8577,7 +8577,7 @@ public final Yield_exprContext yield_expr() throws RecognitionException {
 	}
 
 	public static final String _serializedATN =
-		"\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3h\u067b\4\2\t\2\4"+
+		"\3\u608b\ua72a\u8133\ub9ed\u417c\u3be7\u7786\u5964\3i\u067b\4\2\t\2\4"+
 		"\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+
 		"\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+
 		"\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3Lexer.interp

@@ -106,9 +106,24 @@ def SyntaxError__init__(self, *args, **kwargs):
         self.offset = info[2]
         self.text = info[3]
 
+def SyntaxError__str__(self):
+
+    def isNumber(x):
+        return hasattr(x, "__index__")
+
+    hasLineno = self.lineno and isNumber(self.lineno)
+    if not self.filename and not hasLineno:
+        return "%s" % (self.msg)
+    if self.filename and hasLineno:
+        return "%s (%s, line %d)" % (self.msg, self.filename, self.lineno)
+    if self.filename:
+        return "%s (%s)" % (self.msg, self.filename)
+    return "%s (line %d)" % (self.msg, self.lineno)
 
 SyntaxError.__init__ = SyntaxError__init__
+SyntaxError.__str__ = SyntaxError__str__
 del SyntaxError__init__
+del SyntaxError__str__
 
 
 def UnicodeEncodeError__init__(self, encoding, object, start, end, reason):