pythongh-132661: Implement PEP 750 (python#132662)

Co-authored-by: Lysandros Nikolaou <[email protected]>
Co-authored-by: Bénédikt Tran <[email protected]>
Co-authored-by: Adam Turner <[email protected]>
Co-authored-by: Hugo van Kemenade <[email protected]>
Co-authored-by: Wingy <[email protected]>
Co-authored-by: Koudai Aono <[email protected]>
Co-authored-by: Dave Peck <[email protected]>
Co-authored-by: Terry Jan Reedy <[email protected]>
Co-authored-by: Paul Everitt <[email protected]>
Co-authored-by: sobolevn <[email protected]>

Author: 10 people

.github/CODEOWNERS

@@ -320,3 +320,9 @@ Lib/test/test__colorize.py    @hugovk
 
 # Fuzzing
 Modules/_xxtestfuzz/          @ammaraskar
+
+# t-strings
+**/*interpolationobject*      @lysnikolaou
+**/*templateobject*           @lysnikolaou
+**/*templatelib*              @lysnikolaou
+**/*tstring*                  @lysnikolaou

Doc/library/token.rst

@@ -131,6 +131,41 @@ The token constants are:
 
       The token string contains the closing quote(s).
 
+.. data:: TSTRING_START
+
+   Token value used to indicate the beginning of a template string literal.
+
+   .. impl-detail::
+
+      The token string includes the prefix and the opening quote(s), but none
+      of the contents of the literal.
+
+   .. versionadded:: next
+
+.. data:: TSTRING_MIDDLE
+
+   Token value used for literal text inside a template string literal
+   including format specifications.
+
+   .. impl-detail::
+
+      Replacement fields (that is, the non-literal parts of t-strings) use
+      the same tokens as other expressions, and are delimited by
+      :data:`LBRACE`, :data:`RBRACE`, :data:`EXCLAMATION` and :data:`COLON`
+      tokens.
+
+   .. versionadded:: next
+
+.. data:: TSTRING_END
+
+   Token value used to indicate the end of a template string literal.
+
+   .. impl-detail::
+
+      The token string contains the closing quote(s).
+
+   .. versionadded:: next
+
 .. data:: ENDMARKER
 
    Token value that indicates the end of input.

Doc/whatsnew/3.14.rst

@@ -66,6 +66,7 @@ Summary -- release highlights
 
 * :ref:`PEP 649: deferred evaluation of annotations <whatsnew314-pep649>`
 * :ref:`PEP 741: Python Configuration C API <whatsnew314-pep741>`
+* :ref:`PEP 750: Template Strings <whatsnew314-pep750>`
 * :ref:`PEP 758: Allow except and except* expressions without parentheses <whatsnew314-pep758>`
 * :ref:`PEP 761: Discontinuation of PGP signatures <whatsnew314-pep761>`
 * :ref:`PEP 765: Disallow return/break/continue that exit a finally block <whatsnew314-pep765>`
@@ -92,6 +93,76 @@ If you encounter :exc:`NameError`\s or pickling errors coming out of
 New features
 ============
 
+.. _whatsnew314-pep750:
+
+PEP 750: Template Strings
+-------------------------
+
+Template string literals (t-strings) are a generalization of f-strings,
+using a ``t`` in place of the ``f`` prefix. Instead of evaluating
+to :class:`str`, t-strings evaluate to a new :class:`!string.templatelib.Template` type:
+
+.. code-block:: python
+
+   from string.templatelib import Template
+
+   name = "World"
+   template: Template = t"Hello {name}"
+
+The template can then be combined with functions that operate on the template's
+structure to produce a :class:`str` or a string-like result.
+For example, sanitizing input:
+
+.. code-block:: python
+
+   evil = "<script>alert('evil')</script>"
+   template = t"<p>{evil}</p>"
+   assert html(template) == "<p>&lt;script&gt;alert('evil')&lt;/script&gt;</p>"
+
+As another example, generating HTML attributes from data:
+
+.. code-block:: python
+
+   attributes = {"src": "shrubbery.jpg", "alt": "looks nice"}
+   template = t"<img {attributes} />"
+   assert html(template) == '<img src="shrubbery.jpg" alt="looks nice" class="looks-nice" />'
+
+Unlike f-strings, the ``html`` function has access to template attributes
+containing the original information: static strings, interpolations, and values
+from the original scope. Unlike existing templating approaches, t-strings build
+from the well-known f-string syntax and rules. Template systems thus benefit
+from Python tooling as they are much closer to the Python language, syntax,
+scoping, and more.
+
+Writing template handlers is straightforward:
+
+.. code-block:: python
+
+   from string.templatelib import Template, Interpolation
+
+   def lower_upper(template: Template) -> str:
+       """Render static parts lowercased and interpolations uppercased."""
+       parts: list[str] = []
+       for item in template:
+           if isinstance(item, Interpolation):
+               parts.append(str(item.value).upper())
+           else:
+               parts.append(item.lower())
+       return "".join(parts)
+
+   name = "world"
+   assert lower_upper(t"HELLO {name}") == "hello WORLD"
+
+With this in place, developers can write template systems to sanitize SQL, make
+safe shell operations, improve logging, tackle modern ideas in web development
+(HTML, CSS, and so on), and implement lightweight, custom business DSLs.
+
+See :pep:`750` for more details.
+
+(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono,
+Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran,
+and Pablo Galindo Salgado in :gh:`132661`.)
+
 .. _whatsnew314-pep768:
 
 PEP 768: Safe external debugger interface for CPython

Grammar/Tokens

@@ -62,6 +62,9 @@ SOFT_KEYWORD
 FSTRING_START
 FSTRING_MIDDLE
 FSTRING_END
+TSTRING_START
+TSTRING_MIDDLE
+TSTRING_END
 COMMENT
 NL
 ERRORTOKEN

Grammar/python.gram

@@ -519,7 +519,7 @@ literal_pattern[pattern_ty]:
 literal_expr[expr_ty]:
     | signed_number !('+' | '-')
     | complex_number
-    | strings
+    | &(STRING|FSTRING_START|TSTRING_START) strings
     | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) }
     | 'True' { _PyAST_Constant(Py_True, NULL, EXTRA) }
     | 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) }
@@ -859,7 +859,7 @@ atom[expr_ty]:
     | 'True' { _PyAST_Constant(Py_True, NULL, EXTRA) }
     | 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) }
     | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) }
-    | &(STRING|FSTRING_START) strings
+    | &(STRING|FSTRING_START|TSTRING_START) strings
     | NUMBER
     | &'(' (tuple | group | genexp)
     | &'[' (list | listcomp)
@@ -935,7 +935,7 @@ fstring_middle[expr_ty]:
 fstring_replacement_field[expr_ty]:
     | '{' a=annotated_rhs debug_expr='='? conversion=[fstring_conversion] format=[fstring_full_format_spec] rbrace='}' {
         _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
-    | invalid_replacement_field
+    | invalid_fstring_replacement_field
 fstring_conversion[ResultTokenWithMetadata*]:
     | conv_token="!" conv=NAME { _PyPegen_check_fstring_conversion(p, conv_token, conv) }
 fstring_full_format_spec[ResultTokenWithMetadata*]:
@@ -946,8 +946,27 @@ fstring_format_spec[expr_ty]:
 fstring[expr_ty]:
     | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) }
 
+tstring_format_spec_replacement_field[expr_ty]:
+    | '{' a=annotated_rhs debug_expr='='? conversion=[fstring_conversion] format=[tstring_full_format_spec] rbrace='}' {
+        _PyPegen_formatted_value(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
+    | invalid_tstring_replacement_field
+tstring_format_spec[expr_ty]:
+    | t=TSTRING_MIDDLE { _PyPegen_decoded_constant_from_token(p, t) }
+    | tstring_format_spec_replacement_field
+tstring_full_format_spec[ResultTokenWithMetadata*]:
+    | colon=':' spec=tstring_format_spec* { _PyPegen_setup_full_format_spec(p, colon, (asdl_expr_seq *) spec, EXTRA) }
+tstring_replacement_field[expr_ty]:
+    | '{' a=annotated_rhs debug_expr='='? conversion=[fstring_conversion] format=[tstring_full_format_spec] rbrace='}' {
+        _PyPegen_interpolation(p, a, debug_expr, conversion, format, rbrace, EXTRA) }
+    | invalid_tstring_replacement_field
+tstring_middle[expr_ty]:
+    | tstring_replacement_field
+    | t=TSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) }
+tstring[expr_ty] (memo):
+    | a=TSTRING_START b=tstring_middle* c=TSTRING_END { _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c) }
+
 string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
-strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
+strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
 
 list[expr_ty]:
     | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
@@ -1212,6 +1231,8 @@ invalid_expression:
         RAISE_SYNTAX_ERROR_KNOWN_LOCATION (a, "expected expression before 'if', but statement is given") }
    | a='lambda' [lambda_params] b=':' &FSTRING_MIDDLE  {
         RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") }
+   | a='lambda' [lambda_params] b=':' &TSTRING_MIDDLE  {
+        RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "t-string: lambda expressions are not allowed without parentheses") }
 
 invalid_named_expression(memo):
     | a=expression ':=' expression {
@@ -1454,28 +1475,50 @@ invalid_starred_expression_unpacking:
 invalid_starred_expression:
     | '*' { RAISE_SYNTAX_ERROR("Invalid star expression") }
 
-invalid_replacement_field:
+invalid_fstring_replacement_field:
     | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") }
     | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") }
     | '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before ':'") }
     | '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '}'") }
-    | '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")}
+    | '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'") }
     | '{' annotated_rhs !('=' | '!' | ':' | '}') {
         PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '=', or '!', or ':', or '}'") }
     | '{' annotated_rhs '=' !('!' | ':' | '}') {
         PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '!', or ':', or '}'") }
-    | '{' annotated_rhs '='? invalid_conversion_character
+    | '{' annotated_rhs '='? invalid_fstring_conversion_character
     | '{' annotated_rhs '='? ['!' NAME] !(':' | '}') {
         PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting ':' or '}'") }
     | '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' {
         PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}', or format specs") }
     | '{' annotated_rhs '='? ['!' NAME] !'}' {
         PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}'") }
 
-invalid_conversion_character:
+invalid_fstring_conversion_character:
     | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: missing conversion character") }
     | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: invalid conversion character") }
 
+invalid_tstring_replacement_field:
+    | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '='") }
+    | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '!'") }
+    | '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before ':'") }
+    | '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "t-string: valid expression required before '}'") }
+    | '{' !annotated_rhs { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting a valid expression after '{'") }
+    | '{' annotated_rhs !('=' | '!' | ':' | '}') {
+        PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '=', or '!', or ':', or '}'") }
+    | '{' annotated_rhs '=' !('!' | ':' | '}') {
+        PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '!', or ':', or '}'") }
+    | '{' annotated_rhs '='? invalid_tstring_conversion_character
+    | '{' annotated_rhs '='? ['!' NAME] !(':' | '}') {
+        PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting ':' or '}'") }
+    | '{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}' {
+        PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '}', or format specs") }
+    | '{' annotated_rhs '='? ['!' NAME] !'}' {
+        PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: expecting '}'") }
+
+invalid_tstring_conversion_character:
+    | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
+    | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
+
 invalid_arithmetic:
     | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
 invalid_factor:

Include/internal/pycore_ast.h

@@ -371,6 +371,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(consts)
         STRUCT_FOR_ID(context)
         STRUCT_FOR_ID(contravariant)
+        STRUCT_FOR_ID(conversion)
         STRUCT_FOR_ID(cookie)
         STRUCT_FOR_ID(copy)
         STRUCT_FOR_ID(copyreg)
@@ -428,6 +429,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(exception)
         STRUCT_FOR_ID(existing_file_name)
         STRUCT_FOR_ID(exp)
+        STRUCT_FOR_ID(expression)
         STRUCT_FOR_ID(extend)
         STRUCT_FOR_ID(extra_tokens)
         STRUCT_FOR_ID(facility)