Revert changes in Python-tokenize.c, fix in lexer.c

Author: efimov-mikhail

Lib/test/test_tokenize.py

@@ -3183,8 +3183,7 @@ def get_tokens(string):
             f'__{
                 x:d
             }__'""",
-            "def f():\n if x\n\x00"
-            "class C:\n a\n\x00",
+            " a\n\x00",
         ]:
             with self.subTest(case=case):
                 self.assertRaises(tokenize.TokenError, get_tokens, case)

Parser/lexer/lexer.c

@@ -539,6 +539,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t
                     return MAKE_TOKEN(ERRORTOKEN);
                 }
             }
+            else if (c == EOF && PyErr_Occurred()) {
+                return MAKE_TOKEN(ERRORTOKEN);
+            }
             else {
                 break;
             }

Python/Python-tokenize.c

@@ -251,7 +251,7 @@ tokenizeriter_next(PyObject *op)
 
     int type = _PyTokenizer_Get(it->tok, &token);
     if (type == ERRORTOKEN) {
-        if (!PyErr_Occurred()) {
+        if(!PyErr_Occurred()) {
             _tokenizer_error(it);
             assert(PyErr_Occurred());
         }
@@ -337,10 +337,7 @@ tokenizeriter_next(PyObject *op)
         }
     }
 
-    if (!PyErr_Occurred()) {
-        result = Py_BuildValue("(iN(nn)(nn)O)", type, str, lineno, col_offset, end_lineno, end_col_offset, line);
-    }
-
+    result = Py_BuildValue("(iN(nn)(nn)O)", type, str, lineno, col_offset, end_lineno, end_col_offset, line);
 exit:
     _PyToken_Free(&token);
     if (type == ENDMARKER) {