Small fixes

Author: pablogsal

Grammar/python.gram

@@ -87,7 +87,7 @@ _PyPegen_parse(Parser *p)
 # ==============
 
 file[mod_ty]: a=[statements] ENDMARKER { _PyPegen_make_module(p, a) }
-interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(_PyPegen_register_stmts(p, a), p->arena) }
+interactive[mod_ty]: a=statement_newline { _PyAST_Interactive(a, p->arena) }
 eval[mod_ty]: a=expressions NEWLINE* ENDMARKER { _PyAST_Expression(a, p->arena) }
 func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMARKER { _PyAST_FunctionType(a, b, p->arena) }
 
@@ -100,8 +100,12 @@ statement[asdl_stmt_seq*]:
     | a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } 
     | a[asdl_stmt_seq*]=simple_stmts { a }
 
+single_compound_stmt[asdl_stmt_seq*]:
+    | a=compound_stmt { 
+        _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a)) }
+
 statement_newline[asdl_stmt_seq*]:
-    | a=compound_stmt NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) }
+    | a=single_compound_stmt NEWLINE { a }
     | simple_stmts
     | NEWLINE { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, CHECK(stmt_ty, _PyAST_Pass(EXTRA))) }
     | ENDMARKER { _PyPegen_interactive_exit(p) }

Lib/test/test_syntax.py

@@ -1720,26 +1720,66 @@
 Traceback (most recent call last):
 SyntaxError: invalid syntax. Did you mean 'for'?
 
+>>> for a in b:
+...   pass
+... elso:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'else'?
+
 >>> whille True:
 ...   pass
 Traceback (most recent call last):
 SyntaxError: invalid syntax. Did you mean 'while'?
 
+>>> while True:
+...   pass
+... elso:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'else'?
+
 >>> iff x > 5:
 ...   pass
 Traceback (most recent call last):
 SyntaxError: invalid syntax. Did you mean 'if'?
 
+>>> if x:
+...   pass
+... elseif y:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'elif'?
+
+>>> if x:
+...   pass
+... elif y:
+...   pass
+... elso:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'else'?
+
+>>> tyo:
+...   pass
+... except y:
+...   pass
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'try'?
+
 >>> classe MyClass:
 ...   pass
 Traceback (most recent call last):
 SyntaxError: invalid syntax. Did you mean 'class'?
 
-
 >>> impor math
 Traceback (most recent call last):
 SyntaxError: invalid syntax. Did you mean 'import'?
 
+>>> form x import y
+Traceback (most recent call last):
+SyntaxError: invalid syntax. Did you mean 'from'?
+
 >>> defn calculate_sum(a, b):
 ...   return a + b
 Traceback (most recent call last):

Lib/traceback.py

@@ -1302,7 +1302,7 @@ def _find_keyword_typos(self):
             if self.filename:
                 try:
                     with open(self.filename) as f:
-                        lines = f.readlines()
+                        lines = f.read().splitlines()
                 except Exception:
                     line, end_line, offset = 0,1,0
                 else:
@@ -1313,49 +1313,56 @@ def _find_keyword_typos(self):
 
         error_code = lines[line -1 if line > 0 else 0:end_line]
         error_code[0] = error_code[0][offset:]
-        error_code = textwrap.dedent(''.join(error_code))
+        error_code = textwrap.dedent('\n'.join(error_code))
 
         # Do not continue if the source is too large
         if len(error_code) > 1024:
             return
 
         tokens = tokenize.generate_tokens(io.StringIO(error_code).readline)
         tokens_left_to_process = 10
+        import difflib
         for token in tokens:
-            tokens_left_to_process -= 1
-            if tokens_left_to_process < 0:
-                break
             start, end = token.start, token.end
             if token.type != tokenize.NAME:
                 continue
+            # Only consider NAME tokens on the same line as the error
             if from_filename and token.start[0]+line != end_line+1:
                 continue
             wrong_name = token.string
             if wrong_name in keyword.kwlist:
                 continue
-            suggestion = _suggestions._generate_suggestions(keyword.kwlist, wrong_name)
-            if not suggestion or suggestion == wrong_name:
-                continue
-            # Try to replace the token with the keyword
-            the_lines = error_code.splitlines()
-            the_line = the_lines[start[0] - 1]
-            chars = list(the_line)
-            chars[token.start[1]:token.end[1]] = suggestion
-            the_lines[start[0] - 1] = ''.join(chars)
-            code = ''.join(the_lines)
-            # Check if it works
-            try:
-                codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST)
-            except SyntaxError as e:
-                continue
-            # Keep token.line but handle offsets correctly
-            self.text = token.line
-            self.offset = token.start[1] + 1
-            self.end_offset = token.end[1] + 1
-            self.lineno = start[0]
-            self.end_lineno = end[0]
-            self.msg = f"invalid syntax. Did you mean '{suggestion}'?"
-            return
+
+            # Limit the number of valid tokens to consider to not spend
+            # to much time in this function
+            tokens_left_to_process -= 1
+            if tokens_left_to_process < 0:
+                break
+            # Limit the number of possible matches to try
+            matches = difflib.get_close_matches(wrong_name, keyword.kwlist, n=3)
+            for suggestion in matches:
+                if not suggestion or suggestion == wrong_name:
+                    continue
+                # Try to replace the token with the keyword
+                the_lines = error_code.splitlines()
+                the_line = the_lines[start[0] - 1]
+                chars = list(the_line)
+                chars[token.start[1]:token.end[1]] = suggestion
+                the_lines[start[0] - 1] = ''.join(chars)
+                code = '\n'.join(the_lines)
+                # Check if it works
+                try:
+                    codeop.compile_command(code, symbol="exec", flags=codeop.PyCF_ONLY_AST)
+                except SyntaxError as e:
+                    continue
+                # Keep token.line but handle offsets correctly
+                self.text = token.line
+                self.offset = token.start[1] + 1
+                self.end_offset = token.end[1] + 1
+                self.lineno = start[0]
+                self.end_lineno = end[0]
+                self.msg = f"invalid syntax. Did you mean '{suggestion}'?"
+                return
 
 
     def _format_syntax_error(self, stype, **kwargs):

Parser/action_helpers.c

@@ -1712,17 +1712,11 @@ _PyPegen_checked_future_import(Parser *p, identifier module, asdl_alias_seq * na
     return _PyAST_ImportFrom(module, names, level, lineno, col_offset, end_lineno, end_col_offset, arena);
 }
 
-stmt_ty
-_PyPegen_register_stmt(Parser *p, stmt_ty stmt) {
-    p->last_stmt_location.lineno = stmt->lineno;
-    p->last_stmt_location.col_offset = stmt->col_offset;
-    p->last_stmt_location.end_lineno = stmt->end_lineno;
-    p->last_stmt_location.end_col_offset = stmt->end_col_offset;
-    return stmt;
-}
-
 asdl_stmt_seq*
 _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) {
+    if (!p->call_invalid_rules) {
+        return stmts;
+    }
     Py_ssize_t len = asdl_seq_LEN(stmts);
     if (len == 0) {
         return stmts;
@@ -1733,4 +1727,4 @@ _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) {
     p->last_stmt_location.end_lineno = last_stmt->end_lineno;
     p->last_stmt_location.end_col_offset = last_stmt->end_col_offset;
     return stmts;
-}
+}