Add new tokens TSTRING_MIDDLE & TSTRING_END

Author: lysnikolaou

Doc/library/token.rst

@@ -142,6 +142,30 @@ The token constants are:
 
    .. versionadded:: next
 
+.. data:: TSTRING_MIDDLE
+
+   Token value used for literal text inside a t-string literal
+   including format specifications.
+
+   .. impl-detail::
+
+      Replacement fields (that is, the non-literal parts of f-strings) use
+      the same tokens as other expressions, and are delimited by
+      :data:`LBRACE`, :data:`RBRACE`, :data:`EXCLAMATION` and :data:`COLON`
+      tokens.
+
+   .. versionadded:: next
+
+.. data:: TSTRING_END
+
+   Token value used to indicate the end of a t-string.
+
+   .. impl-detail::
+
+      The token string contains the closing quote(s).
+
+   .. versionadded:: next
+
 .. data:: ENDMARKER
 
    Token value that indicates the end of input.

Grammar/Tokens

@@ -60,9 +60,11 @@ TYPE_IGNORE
 TYPE_COMMENT
 SOFT_KEYWORD
 FSTRING_START
-TSTRING_START
 FSTRING_MIDDLE
 FSTRING_END
+TSTRING_START
+TSTRING_MIDDLE
+TSTRING_END
 COMMENT
 NL
 ERRORTOKEN

Grammar/python.gram

@@ -945,7 +945,7 @@ tstring_format_spec_replacement_field[expr_ty]:
         _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
     | invalid_tstring_replacement_field
 tstring_format_spec[expr_ty]:
-    | t=FSTRING_MIDDLE { _PyPegen_decoded_constant_from_token(p, t) }
+    | t=TSTRING_MIDDLE { _PyPegen_decoded_constant_from_token(p, t) }
     | tstring_format_spec_replacement_field
 tstring_full_format_spec[ResultTokenWithMetadata*]:
     | colon=':' spec=tstring_format_spec* { _PyPegen_setup_full_format_spec(p, colon, (asdl_expr_seq *) spec, EXTRA) }
@@ -955,9 +955,9 @@ tstring_replacement_field[expr_ty]:
     | invalid_tstring_replacement_field
 tstring_middle[expr_ty]:
     | tstring_replacement_field
-    | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
+    | t=TSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
 tstring[expr_ty] (memo):
-    | a=TSTRING_START b=tstring_middle* c=FSTRING_END { _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c) }
+    | a=TSTRING_START b=tstring_middle* c=TSTRING_END { _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c) }
 
 string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
 strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
@@ -1224,8 +1224,9 @@ invalid_expression:
    | a[stmt_ty]=(pass_stmt|break_stmt|continue_stmt) 'if' b=disjunction 'else' c=simple_stmt {
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION (a, "expected expression before 'if', but statement is given") }
    | a='lambda' [lambda_params] b=':' &FSTRING_MIDDLE  {
-        RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "%c-string: lambda expressions are not allowed without parentheses",
-            TOK_GET_STRING_PREFIX(p->tok)) }
+        RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") }
+   | a='lambda' [lambda_params] b=':' &TSTRING_MIDDLE  {
+        RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "t-string: lambda expressions are not allowed without parentheses") }
 
 invalid_named_expression(memo):
     | a=expression ':=' expression {

Include/internal/pycore_token.h

@@ -73,13 +73,15 @@ extern "C" {
 #define TYPE_COMMENT    57
 #define SOFT_KEYWORD    58
 #define FSTRING_START   59
-#define TSTRING_START   60
-#define FSTRING_MIDDLE  61
-#define FSTRING_END     62
-#define COMMENT         63
-#define NL              64
-#define ERRORTOKEN      65
-#define N_TOKENS        67
+#define FSTRING_MIDDLE  60
+#define FSTRING_END     61
+#define TSTRING_START   62
+#define TSTRING_MIDDLE  63
+#define TSTRING_END     64
+#define COMMENT         65
+#define NL              66
+#define ERRORTOKEN      67
+#define N_TOKENS        69
 #define NT_OFFSET       256
 
 /* Special definitions for cooperation with parser */
@@ -92,7 +94,8 @@ extern "C" {
                                  (x) == INDENT    || \
                                  (x) == DEDENT)
 #define ISSTRINGLIT(x)          ((x) == STRING           || \
-                                 (x) == FSTRING_MIDDLE)
+                                 (x) == FSTRING_MIDDLE   || \
+                                 (x) == TSTRING_MIDDLE)
 
 
 // Export these 4 symbols for 'test_peg_generator'

Lib/token.py

@@ -251,7 +251,7 @@ def untokenize(self, iterable):
                     self.tokens.append(indent)
                     self.prev_col = len(indent)
                 startline = False
-            elif tok_type == FSTRING_MIDDLE:
+            elif tok_type in (FSTRING_MIDDLE, TSTRING_MIDDLE):
                 if '{' in token or '}' in token:
                     token = self.escape_brackets(token)
                     last_line = token.splitlines()[-1]
@@ -308,7 +308,7 @@ def compat(self, token, iterable):
             elif startline and indents:
                 toks_append(indents[-1])
                 startline = False
-            elif toknum == FSTRING_MIDDLE:
+            elif toknum in (FSTRING_MIDDLE, TSTRING_MIDDLE):
                 tokval = self.escape_brackets(tokval)
 
             # Insert a space between two consecutive brackets if we are in an f-string

Lib/tokenize.py

@@ -38,6 +38,8 @@ static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) {
 #define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index]))
 #endif
 
+#define FTSTRING_MIDDLE(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_MIDDLE : FSTRING_MIDDLE)
+#define FTSTRING_END(tok_mode) (tok_mode->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
 #define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
 #define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end)
 #define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\
@@ -1335,7 +1337,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
     p_start = tok->start;
     p_end = tok->cur;
     tok->tok_mode_stack_index--;
-    return MAKE_TOKEN(FSTRING_END);
+    return MAKE_TOKEN(FTSTRING_END(current_tok));
 
 f_string_middle:
 
@@ -1375,7 +1377,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
                 current_tok->in_format_spec = 0;
                 p_start = tok->start;
                 p_end = tok->cur;
-                return MAKE_TOKEN(FSTRING_MIDDLE);
+                return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
             }
 
             assert(tok->multi_line_start != NULL);
@@ -1435,12 +1437,12 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
                 p_start = tok->start;
                 p_end = tok->cur - 1;
             }
-            return MAKE_TOKEN(FSTRING_MIDDLE);
+            return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
         } else if (c == '}') {
             if (unicode_escape) {
                 p_start = tok->start;
                 p_end = tok->cur;
-                return MAKE_TOKEN(FSTRING_MIDDLE);
+                return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
             }
             int peek = tok_nextc(tok);
 
@@ -1460,7 +1462,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
                 p_start = tok->start;
                 p_end = tok->cur;
             }
-            return MAKE_TOKEN(FSTRING_MIDDLE);
+            return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
         } else if (c == '\\') {
             int peek = tok_nextc(tok);
             if (peek == '\r') {
@@ -1502,7 +1504,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct
     }
     p_start = tok->start;
     p_end = tok->cur;
-    return MAKE_TOKEN(FSTRING_MIDDLE);
+    return MAKE_TOKEN(FTSTRING_MIDDLE(current_tok));
 }
 
 static int