fixup! gh-137314: Fix incorrect treatment of format specs in raw fstrings (GH-137328) (cherry picked from commit 0153d82)

Author: pablogsal

Parser/action_helpers.c

@@ -1,6 +1,7 @@
 #include <Python.h>
 
 #include "pegen.h"
+#include "lexer/state.h"
 #include "string_parser.h"
 #include "pycore_runtime.h"         // _PyRuntime
 #include "pycore_pystate.h"         // _PyInterpreterState_GET()
@@ -1374,7 +1375,7 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
     int is_raw = 0;
     if (INSIDE_FSTRING(p->tok)) {
         tokenizer_mode *mode = TOK_GET_MODE(p->tok);
-        is_raw = mode->raw;
+        is_raw = mode->f_string_raw;
     }
 
     PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);

Parser/lexer/lexer.c

@@ -22,22 +22,6 @@
                || c == '_'\
                || (c >= 128))
 
-#ifdef Py_DEBUG
-static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) {
-    assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
-    return &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
-}
-static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
-    assert(tok->tok_mode_stack_index >= 0);
-    assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
-    return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
-}
-#else
-#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
-#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index]))
-#endif
-
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 #define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
                 _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end))

Parser/lexer/state.h

@@ -1,6 +1,7 @@
 #ifndef _PY_LEXER_H_
 #define _PY_LEXER_H_
 
+#include "Python.h"
 #include "object.h"
 
 #define MAXINDENT 100       /* Max indentation level */
@@ -138,5 +139,20 @@ void _PyTokenizer_Free(struct tok_state *);
 void _PyToken_Free(struct token *);
 void _PyToken_Init(struct token *);
 
+#ifdef Py_DEBUG
+static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) {
+    assert(tok->tok_mode_stack_index >= 0);
+    assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL);
+    return &(tok->tok_mode_stack[tok->tok_mode_stack_index]);
+}
+static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
+    assert(tok->tok_mode_stack_index >= 0);
+    assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL);
+    return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]);
+}
+#else
+#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index]))
+#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index]))
+#endif
 
 #endif