pythongh-105820: Fix tok_mode expression buffer in file & readline tokenizer (python#105828)

Author: lysnikolaou

Lib/test/test_fstring.py

@@ -15,7 +15,7 @@
 import unittest
 from test import support
 from test.support.os_helper import temp_cwd
-from test.support.script_helper import assert_python_failure
+from test.support.script_helper import assert_python_failure, assert_python_ok
 
 a_global = 'global variable'
 
@@ -1635,5 +1635,18 @@ def test_syntax_error_after_debug(self):
                                 "f'{1=}{1;}'",
                             ])
 
+    def test_debug_in_file(self):
+        with temp_cwd():
+            script = 'script.py'
+            with open('script.py', 'w') as f:
+                f.write(f"""\
+print(f'''{{
+3
+=}}''')""")
+
+            _, stdout, _ = assert_python_ok(script)
+        self.assertEqual(stdout.decode('utf-8').strip().replace('\r\n', '\n').replace('\r', '\n'),
+                         "3\n=3")
+
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_tokenize.py

@@ -558,6 +558,19 @@ def test_string(self):
     OP         '}'           (1, 39) (1, 40)
     FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
     FSTRING_END "'"           (1, 52) (1, 53)
+    """)
+        self.check_tokenize("""\
+f'''{
+3
+=}'''""", """\
+    FSTRING_START "f'''"        (1, 0) (1, 4)
+    OP         '{'           (1, 4) (1, 5)
+    NL         '\\n'          (1, 5) (1, 6)
+    NUMBER     '3'           (2, 0) (2, 1)
+    NL         '\\n'          (2, 1) (2, 2)
+    OP         '='           (3, 0) (3, 1)
+    OP         '}'           (3, 1) (3, 2)
+    FSTRING_END "'''"         (3, 2) (3, 5)
     """)
 
     def test_function(self):

Misc/NEWS.d/next/Core and Builtins/2023-06-15-15-54-47.gh-issue-105831.-MC9Zs.rst

@@ -0,0 +1 @@
+Fix an f-string bug, where using a debug expression (the `=` sign) that appears in the last line of a file results to the debug buffer that holds the expression text being one character too small.

Parser/tokenizer.c

@@ -1039,9 +1039,6 @@ tok_readline_raw(struct tok_state *tok)
         if (line == NULL) {
             return 1;
         }
-        if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
-            return 0;
-        }
         if (tok->fp_interactive &&
             tok_concatenate_interactive_new_line(tok, line) == -1) {
             return 0;
@@ -1270,6 +1267,10 @@ tok_underflow_file(struct tok_state *tok) {
         tok->implicit_newline = 1;
     }
 
+    if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
+        return 0;
+    }
+
     ADVANCE_LINENO();
     if (tok->decoding_state != STATE_NORMAL) {
         if (tok->lineno > 2) {
@@ -1314,6 +1315,10 @@ tok_underflow_readline(struct tok_state* tok) {
         tok->implicit_newline = 1;
     }
 
+    if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) {
+        return 0;
+    }
+
     ADVANCE_LINENO();
     /* The default encoding is UTF-8, so make sure we don't have any
        non-UTF-8 sequences in it. */