[3.14] pythongh-133541: Handle SyntaxError raised by the tokenizer on user input (pythonGH-133606) (python#133709)

Author: miss-islington

Lib/_pyrepl/utils.py

@@ -102,6 +102,8 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]:
         for color in gen_colors_from_token_stream(gen, line_lengths):
             yield color
             last_emitted = color
+    except SyntaxError:
+        return
     except tokenize.TokenError as te:
         yield from recover_unterminated_string(
             te, line_lengths, last_emitted, buffer

Lib/test/test_pyrepl/test_reader.py

@@ -497,6 +497,26 @@ def unfinished_function():
         self.assert_screen_equal(reader, code, clean=True)
         self.assert_screen_equal(reader, expected)
 
+    def test_syntax_highlighting_indentation_error(self):
+        code = dedent(
+            """\
+            def unfinished_function():
+                var = 1
+               oops
+            """
+        )
+        expected = dedent(
+            """\
+            {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z}
+                var {o}={z} {n}1{z}
+               oops
+            """
+        ).format(**colors)
+        events = code_to_events(code)
+        reader, _ = handle_all_events(events)
+        self.assert_screen_equal(reader, code, clean=True)
+        self.assert_screen_equal(reader, expected)
+
     def test_control_characters(self):
         code = 'flag = "🏳️‍🌈"'
         events = code_to_events(code)

Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst

@@ -0,0 +1,2 @@
+Inconsistent indentation in user input crashed the new REPL when syntax
+highlighting was active. This is now fixed.