[3.14] pythongh-132661: Disallow Template/str concatenation after PEP 750 spec update (python#135996) (python#136901)

Co-authored-by: Dave Peck <[email protected]>
Co-authored-by: sobolevn <[email protected]>

Author: 3 people

Grammar/python.gram

@@ -985,7 +985,10 @@ tstring[expr_ty] (memo):
             _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) }
 
 string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) }
-strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
+strings[expr_ty] (memo):
+    | invalid_string_tstring_concat
+    | a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) }
+    | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) }
 
 list[expr_ty]:
     | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) }
@@ -1546,6 +1549,12 @@ invalid_tstring_conversion_character:
     | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") }
     | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") }
 
+invalid_string_tstring_concat:
+    | a=(fstring|string)+ b[expr_ty]=tstring {
+        RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
+    | a=tstring+ b[expr_ty]=(fstring|string) {
+        RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") }
+
 invalid_arithmetic:
     | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") }
 invalid_factor:

Lib/_ast_unparse.py

@@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix):
             )
         self._ftstring_helper(fstring_parts)
 
-    def _tstring_helper(self, node):
-        if not node.values:
-            self._write_ftstring([], "t")
-            return
-        last_idx = 0
-        for i, value in enumerate(node.values):
-            # This can happen if we have an implicit concat of a t-string
-            # with an f-string
-            if isinstance(value, FormattedValue):
-                if i > last_idx:
-                    # Write t-string until here
-                    self._write_ftstring(node.values[last_idx:i], "t")
-                    self.write(" ")
-                # Write f-string with the current formatted value
-                self._write_ftstring([node.values[i]], "f")
-                if i + 1 < len(node.values):
-                    # Only add a space if there are more values after this
-                    self.write(" ")
-                last_idx = i + 1
-
-        if last_idx < len(node.values):
-            # Write t-string from last_idx to end
-            self._write_ftstring(node.values[last_idx:], "t")
-
     def visit_JoinedStr(self, node):
         self._write_ftstring(node.values, "f")
 
     def visit_TemplateStr(self, node):
-        self._tstring_helper(node)
+        self._write_ftstring(node.values, "t")
 
     def _write_ftstring_inner(self, node, is_format_spec=False):
         if isinstance(node, JoinedStr):

Lib/test/test_ast/test_ast.py

@@ -999,13 +999,6 @@ def test_tstring(self):
         self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
         self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
 
-        # Test AST for implicit concat of t-string with f-string
-        tree = ast.parse('t"Hello {name}" f"{name}"')
-        self.assertIsInstance(tree.body[0].value, ast.TemplateStr)
-        self.assertIsInstance(tree.body[0].value.values[0], ast.Constant)
-        self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation)
-        self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue)
-
 
 class CopyTests(unittest.TestCase):
     """Test copying and pickling AST nodes."""

Lib/test/test_tstring.py

@@ -150,7 +150,6 @@ def test_raw_tstrings(self):
         t = tr"{path}\Documents"
         self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")])
 
-
     def test_template_concatenation(self):
         # Test template + template
         t1 = t"Hello, "
@@ -161,9 +160,10 @@ def test_template_concatenation(self):
 
         # Test template + string
         t1 = t"Hello"
-        combined = t1 + ", world"
-        self.assertTStringEqual(combined, ("Hello, world",), ())
-        self.assertEqual(fstring(combined), "Hello, world")
+        expected_msg = 'can only concatenate string.templatelib.Template ' \
+            '\\(not "str"\\) to string.templatelib.Template'
+        with self.assertRaisesRegex(TypeError, expected_msg):
+            t1 + ", world"
 
         # Test template + template with interpolation
         name = "Python"
@@ -174,9 +174,10 @@ def test_template_concatenation(self):
         self.assertEqual(fstring(combined), "Hello, Python")
 
         # Test string + template
-        t = "Hello, " + t"{name}"
-        self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
-        self.assertEqual(fstring(t), "Hello, Python")
+        expected_msg = 'can only concatenate str ' \
+            '\\(not "string.templatelib.Template"\\) to str'
+        with self.assertRaisesRegex(TypeError, expected_msg):
+            "Hello, " + t"{name}"
 
     def test_nested_templates(self):
         # Test a template inside another template expression
@@ -241,52 +242,28 @@ def test_literal_concatenation(self):
         self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")])
         self.assertEqual(fstring(t), "Hello, Python")
 
-        # Test concatenation with string literal
-        name = "Python"
-        t = t"Hello, {name}" "and welcome!"
-        self.assertTStringEqual(
-            t, ("Hello, ", "and welcome!"), [(name, "name")]
-        )
-        self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
-
-        # Test concatenation with Unicode literal
-        name = "Python"
-        t = t"Hello, {name}" u"and welcome!"
-        self.assertTStringEqual(
-            t, ("Hello, ", "and welcome!"), [(name, "name")]
-        )
-        self.assertEqual(fstring(t), "Hello, Pythonand welcome!")
-
-        # Test concatenation with f-string literal
-        tab = '\t'
-        t = t"Tab: {tab}. " f"f-tab: {tab}."
-        self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")])
-        self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.")
-
-        # Test concatenation with raw string literal
-        tab = '\t'
-        t = t"Tab: {tab}. " r"Raw tab: \t."
-        self.assertTStringEqual(
-            t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")]
-        )
-        self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.")
-
-        # Test concatenation with raw f-string literal
-        tab = '\t'
-        t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t."
-        self.assertTStringEqual(
-            t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")]
-        )
-        self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.")
-
+        # Test disallowed mix of t-string and string/f-string (incl. bytes)
         what = 't'
-        expected_msg = 'cannot mix bytes and nonbytes literals'
+        expected_msg = 'cannot mix t-string literals with string or bytes literals'
         for case in (
+            "t'{what}-string literal' 'str literal'",
+            "t'{what}-string literal' u'unicode literal'",
+            "t'{what}-string literal' f'f-string literal'",
+            "t'{what}-string literal' r'raw string literal'",
+            "t'{what}-string literal' rf'raw f-string literal'",
             "t'{what}-string literal' b'bytes literal'",
             "t'{what}-string literal' br'raw bytes literal'",
+            "'str literal' t'{what}-string literal'",
+            "u'unicode literal' t'{what}-string literal'",
+            "f'f-string literal' t'{what}-string literal'",
+            "r'raw string literal' t'{what}-string literal'",
+            "rf'raw f-string literal' t'{what}-string literal'",
+            "b'bytes literal' t'{what}-string literal'",
+            "br'raw bytes literal' t'{what}-string literal'",
         ):
-            with self.assertRaisesRegex(SyntaxError, expected_msg):
-                eval(case)
+            with self.subTest(case):
+                with self.assertRaisesRegex(SyntaxError, expected_msg):
+                    eval(case)
 
     def test_triple_quoted(self):
         # Test triple-quoted t-strings

Lib/test/test_unparse.py

@@ -206,10 +206,6 @@ def test_tstrings(self):
         self.check_ast_roundtrip("t'foo'")
         self.check_ast_roundtrip("t'foo {bar}'")
         self.check_ast_roundtrip("t'foo {bar!s:.2f}'")
-        self.check_ast_roundtrip("t'foo {bar}' f'{bar}'")
-        self.check_ast_roundtrip("f'{bar}' t'foo {bar}'")
-        self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'")
-        self.check_ast_roundtrip("t'foo {bar}' u'bar'")
 
     def test_strings(self):
         self.check_ast_roundtrip("u'foo'")

Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst

@@ -0,0 +1,5 @@
+Reflect recent :pep:`750` change.
+
+Disallow concatenation of ``string.templatelib.Template`` and :class:`str`.
+Also, disallow implicit concatenation of t-string literals with string or
+f-string literals.

Objects/templateobject.c

@@ -30,7 +30,8 @@ templateiter_next(PyObject *op)
             Py_SETREF(item, PyIter_Next(self->interpolationsiter));
             self->from_strings = 1;
         }
-    } else {
+    }
+    else {
         item = PyIter_Next(self->interpolationsiter);
         self->from_strings = 1;
     }
@@ -245,54 +246,6 @@ template_iter(PyObject *op)
     return (PyObject *)iter;
 }
 
-static PyObject *
-template_strings_append_str(PyObject *strings, PyObject *str)
-{
-    Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
-    PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1);
-    PyObject *concat = PyUnicode_Concat(string, str);
-    if (concat == NULL) {
-        return NULL;
-    }
-
-    PyObject *newstrings = PyTuple_New(stringslen);
-    if (newstrings == NULL) {
-        Py_DECREF(concat);
-        return NULL;
-    }
-
-    for (Py_ssize_t i = 0; i < stringslen - 1; i++) {
-        PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
-    }
-    PyTuple_SET_ITEM(newstrings, stringslen - 1, concat);
-
-    return newstrings;
-}
-
-static PyObject *
-template_strings_prepend_str(PyObject *strings, PyObject *str)
-{
-    Py_ssize_t stringslen = PyTuple_GET_SIZE(strings);
-    PyObject *string = PyTuple_GET_ITEM(strings, 0);
-    PyObject *concat = PyUnicode_Concat(str, string);
-    if (concat == NULL) {
-        return NULL;
-    }
-
-    PyObject *newstrings = PyTuple_New(stringslen);
-    if (newstrings == NULL) {
-        Py_DECREF(concat);
-        return NULL;
-    }
-
-    PyTuple_SET_ITEM(newstrings, 0, concat);
-    for (Py_ssize_t i = 1; i < stringslen; i++) {
-        PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i)));
-    }
-
-    return newstrings;
-}
-
 static PyObject *
 template_strings_concat(PyObject *left, PyObject *right)
 {
@@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other)
     return newtemplate;
 }
 
-static PyObject *
-template_concat_template_str(templateobject *self, PyObject *other)
-{
-    PyObject *newstrings = template_strings_append_str(self->strings, other);
-    if (newstrings == NULL) {
-        return NULL;
-    }
-
-    PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
-    Py_DECREF(newstrings);
-    return newtemplate;
-}
-
-static PyObject *
-template_concat_str_template(templateobject *self, PyObject *other)
-{
-    PyObject *newstrings = template_strings_prepend_str(self->strings, other);
-    if (newstrings == NULL) {
-        return NULL;
-    }
-
-    PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations);
-    Py_DECREF(newstrings);
-    return newtemplate;
-}
-
 PyObject *
 _PyTemplate_Concat(PyObject *self, PyObject *other)
 {
     if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) {
         return template_concat_templates((templateobject *) self, (templateobject *) other);
     }
-    else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) {
-        return template_concat_template_str((templateobject *) self, other);
-    }
-    else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) {
-        return template_concat_str_template((templateobject *) other, self);
-    }
-    else {
-        Py_RETURN_NOTIMPLEMENTED;
-    }
+
+    PyErr_Format(PyExc_TypeError,
+        "can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template",
+        other);
+    return NULL;
 }
 
 static PyObject *

Objects/unicodeobject.c

@@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 #include "pycore_pyhash.h"        // _Py_HashSecret_t
 #include "pycore_pylifecycle.h"   // _Py_SetFileSystemEncoding()
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#include "pycore_template.h"      // _PyTemplate_Concat()
 #include "pycore_tuple.h"         // _PyTuple_FromArray()
 #include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
@@ -11639,16 +11638,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
         return NULL;
 
     if (!PyUnicode_Check(right)) {
-        if (_PyTemplate_CheckExact(right)) {
-            // str + tstring is implemented in the tstring type
-            return _PyTemplate_Concat(left, right);
-        }
-        else {
-            PyErr_Format(PyExc_TypeError,
-                "can only concatenate str (not \"%.200s\") to str",
-                Py_TYPE(right)->tp_name);
-            return NULL;
-        }
+        PyErr_Format(PyExc_TypeError,
+            "can only concatenate str (not \"%.200s\") to str",
+            Py_TYPE(right)->tp_name);
+        return NULL;
     }
 
     /* Shortcuts */