pythongh-119311: Fix name mangling with PEP 695 generic classes (python#119464)

JelleZijlstra · estyxx · commit c16552fd28bb · 2024-07-17T09:51:30.000+01:00
Fixes python#119311. Fixes python#119395.
diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h
@@ -81,6 +81,7 @@ typedef struct _symtable_entry {
     PyObject *ste_varnames;  /* list of function parameters */
     PyObject *ste_children;  /* list of child blocks */
     PyObject *ste_directives;/* locations of global and nonlocal statements */
+    PyObject *ste_mangled_names; /* set of names for which mangling should be applied */
     _Py_block_ty ste_type;
     int ste_nested;      /* true if block is nested */
     unsigned ste_free : 1;        /* true if block has free variables */
@@ -128,6 +129,7 @@ extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *);
 
 extern void _PySymtable_Free(struct symtable *);
 
+extern PyObject *_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name);
 extern PyObject* _Py_Mangle(PyObject *p, PyObject *name);
 
 /* Flags for def-use information */
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
@@ -473,6 +473,7 @@ def _write_atomic(path, data, mode=0o666):
 #     Python 3.13a5 3569 (Specialize CONTAINS_OP)
 #     Python 3.13a6 3570 (Add __firstlineno__ class attribute)
 #     Python 3.14a1 3600 (Add LOAD_COMMON_CONSTANT)
+#     Python 3.14a1 3601 (Fix miscompilation of private names in generic classes)
 
 #     Python 3.15 will start with 3700
 
@@ -489,7 +490,7 @@ def _write_atomic(path, data, mode=0o666):
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3600).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3601).to_bytes(2, 'little') + b'\r\n'
 
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py
@@ -823,6 +823,100 @@ def meth[__U](self, arg: __T, arg2: __U):
 
         self.assertEqual(Foo.Alias.__value__, (T, V))
 
+    def test_no_leaky_mangling_in_module(self):
+        ns = run_code("""
+            __before = "before"
+            class X[T]: pass
+            __after = "after"
+        """)
+        self.assertEqual(ns["__before"], "before")
+        self.assertEqual(ns["__after"], "after")
+
+    def test_no_leaky_mangling_in_function(self):
+        ns = run_code("""
+            def f():
+                class X[T]: pass
+                _X_foo = 2
+                __foo = 1
+                assert locals()['__foo'] == 1
+                return __foo
+        """)
+        self.assertEqual(ns["f"](), 1)
+
+    def test_no_leaky_mangling_in_class(self):
+        ns = run_code("""
+            class Outer:
+                __before = "before"
+                class Inner[T]:
+                    __x = "inner"
+                __after = "after"
+        """)
+        Outer = ns["Outer"]
+        self.assertEqual(Outer._Outer__before, "before")
+        self.assertEqual(Outer.Inner._Inner__x, "inner")
+        self.assertEqual(Outer._Outer__after, "after")
+
+    def test_no_mangling_in_bases(self):
+        ns = run_code("""
+            class __Base:
+                def __init_subclass__(self, **kwargs):
+                    self.kwargs = kwargs
+
+            class Derived[T](__Base, __kwarg=1):
+                pass
+        """)
+        Derived = ns["Derived"]
+        self.assertEqual(Derived.__bases__, (ns["__Base"], Generic))
+        self.assertEqual(Derived.kwargs, {"__kwarg": 1})
+
+    def test_no_mangling_in_nested_scopes(self):
+        ns = run_code("""
+            from test.test_type_params import make_base
+
+            class __X:
+                pass
+
+            class Y[T: __X](
+                make_base(lambda: __X),
+                # doubly nested scope
+                make_base(lambda: (lambda: __X)),
+                # list comprehension
+                make_base([__X for _ in (1,)]),
+                # genexp
+                make_base(__X for _ in (1,)),
+            ):
+                pass
+        """)
+        Y = ns["Y"]
+        T, = Y.__type_params__
+        self.assertIs(T.__bound__, ns["__X"])
+        base0 = Y.__bases__[0]
+        self.assertIs(base0.__arg__(), ns["__X"])
+        base1 = Y.__bases__[1]
+        self.assertIs(base1.__arg__()(), ns["__X"])
+        base2 = Y.__bases__[2]
+        self.assertEqual(base2.__arg__, [ns["__X"]])
+        base3 = Y.__bases__[3]
+        self.assertEqual(list(base3.__arg__), [ns["__X"]])
+
+    def test_type_params_are_mangled(self):
+        ns = run_code("""
+            from test.test_type_params import make_base
+
+            class Foo[__T, __U: __T](make_base(__T), make_base(lambda: __T)):
+                param = __T
+        """)
+        Foo = ns["Foo"]
+        T, U = Foo.__type_params__
+        self.assertEqual(T.__name__, "__T")
+        self.assertEqual(U.__name__, "__U")
+        self.assertIs(U.__bound__, T)
+        self.assertIs(Foo.param, T)
+
+        base1, base2, *_ = Foo.__bases__
+        self.assertIs(base1.__arg__, T)
+        self.assertIs(base2.__arg__(), T)
+
 
 class TypeParamsComplexCallsTest(unittest.TestCase):
     def test_defaults(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-14.gh-issue-119395.z-Hsqb.rst
@@ -0,0 +1,2 @@
+Fix bug where names appearing after a generic class are mangled as if they
+are in the generic class.
diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-06-34-45.gh-issue-119311.2DBwKR.rst
@@ -0,0 +1,2 @@
+Fix bug where names are unexpectedly mangled in the bases of generic
+classes.
diff --git a/Python/compile.c b/Python/compile.c
@@ -1001,7 +1001,7 @@ static int
 compiler_addop_name(struct compiler_unit *u, location loc,
                     int opcode, PyObject *dict, PyObject *o)
 {
-    PyObject *mangled = _Py_Mangle(u->u_private, o);
+    PyObject *mangled = _Py_MaybeMangle(u->u_private, u->u_ste, o);
     if (!mangled) {
         return ERROR;
     }
@@ -1873,7 +1873,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc,
         arg_ty arg = asdl_seq_GET(kwonlyargs, i);
         expr_ty default_ = asdl_seq_GET(kw_defaults, i);
         if (default_) {
-            PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg);
+            PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, arg->arg);
             if (!mangled) {
                 goto error;
             }
@@ -1930,7 +1930,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,
     if (!annotation) {
         return SUCCESS;
     }
-    PyObject *mangled = _Py_Mangle(c->u->u_private, id);
+    PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, id);
     if (!mangled) {
         return ERROR;
     }
@@ -2625,7 +2625,6 @@ compiler_class(struct compiler *c, stmt_ty s)
     asdl_type_param_seq *type_params = s->v.ClassDef.type_params;
     int is_generic = asdl_seq_LEN(type_params) > 0;
     if (is_generic) {
-        Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
         PyObject *type_params_name = PyUnicode_FromFormat("<generic parameters of %U>",
                                                          s->v.ClassDef.name);
         if (!type_params_name) {
@@ -2637,6 +2636,7 @@ compiler_class(struct compiler *c, stmt_ty s)
             return ERROR;
         }
         Py_DECREF(type_params_name);
+        Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));
         RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params));
         _Py_DECLARE_STR(type_params, ".type_params");
         RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store));
@@ -4203,7 +4203,7 @@ compiler_nameop(struct compiler *c, location loc,
         return ERROR;
     }
 
-    mangled = _Py_Mangle(c->u->u_private, name);
+    mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name);
     if (!mangled) {
         return ERROR;
     }
@@ -6512,7 +6512,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)
                 VISIT(c, expr, s->v.AnnAssign.annotation);
             }
             ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);
-            mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id);
+            mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);
             ADDOP_LOAD_CONST_NEW(c, loc, mangled);
             ADDOP(c, loc, STORE_SUBSCR);
         }
diff --git a/Python/symtable.c b/Python/symtable.c
@@ -103,6 +103,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block,
     ste->ste_children = NULL;
 
     ste->ste_directives = NULL;
+    ste->ste_mangled_names = NULL;
 
     ste->ste_type = block;
     ste->ste_nested = 0;
@@ -166,6 +167,7 @@ ste_dealloc(PySTEntryObject *ste)
     Py_XDECREF(ste->ste_varnames);
     Py_XDECREF(ste->ste_children);
     Py_XDECREF(ste->ste_directives);
+    Py_XDECREF(ste->ste_mangled_names);
     PyObject_Free(ste);
 }
 
@@ -1338,6 +1340,11 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
     if (prev) {
         ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
     }
+    /* No need to inherit ste_mangled_names in classes, where all names
+     * are mangled. */
+    if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) {
+        ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names);
+    }
     /* The entry is owned by the stack. Borrow it for st_cur. */
     Py_DECREF(ste);
     st->st_cur = ste;
@@ -1363,7 +1370,7 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
 static long
 symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name)
 {
-    PyObject *mangled = _Py_Mangle(st->st_private, name);
+    PyObject *mangled = _Py_MaybeMangle(st->st_private, ste, name);
     if (!mangled)
         return 0;
     long ret = _PyST_GetSymbol(ste, mangled);
@@ -1384,8 +1391,7 @@ symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _s
     PyObject *o;
     PyObject *dict;
     long val;
-    PyObject *mangled = _Py_Mangle(st->st_private, name);
-
+    PyObject *mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
 
     if (!mangled)
         return 0;
@@ -1474,6 +1480,11 @@ static int
 symtable_add_def(struct symtable *st, PyObject *name, int flag,
                  int lineno, int col_offset, int end_lineno, int end_col_offset)
 {
+    if ((flag & DEF_TYPE_PARAM) && st->st_cur->ste_mangled_names != NULL) {
+        if(PySet_Add(st->st_cur->ste_mangled_names, name) < 0) {
+            return 0;
+        }
+    }
     return symtable_add_def_helper(st, name, flag, st->st_cur,
                         lineno, col_offset, end_lineno, end_col_offset);
 }
@@ -1508,7 +1519,6 @@ symtable_enter_type_param_block(struct symtable *st, identifier name,
                               lineno, col_offset, end_lineno, end_col_offset)) {
             return 0;
         }
-        st->st_private = name;
         // This is used for setting the generic base
         _Py_DECLARE_STR(generic_base, ".generic_base");
         if (!symtable_add_def(st, &_Py_STR(generic_base), DEF_LOCAL,
@@ -1597,7 +1607,7 @@ symtable_record_directive(struct symtable *st, identifier name, int lineno,
         if (!st->st_cur->ste_directives)
             return 0;
     }
-    mangled = _Py_Mangle(st->st_private, name);
+    mangled = _Py_MaybeMangle(st->st_private, st->st_cur, name);
     if (!mangled)
         return 0;
     data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
@@ -1673,13 +1683,19 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             VISIT_QUIT(st, 0);
         if (s->v.ClassDef.decorator_list)
             VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
+        tmp = st->st_private;
         if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
             if (!symtable_enter_type_param_block(st, s->v.ClassDef.name,
                                                 (void *)s->v.ClassDef.type_params,
                                                 false, false, s->kind,
                                                 LOCATION(s))) {
                 VISIT_QUIT(st, 0);
             }
+            st->st_private = s->v.ClassDef.name;
+            st->st_cur->ste_mangled_names = PySet_New(NULL);
+            if (!st->st_cur->ste_mangled_names) {
+                VISIT_QUIT(st, 0);
+            }
             VISIT_SEQ(st, type_param, s->v.ClassDef.type_params);
         }
         VISIT_SEQ(st, expr, s->v.ClassDef.bases);
@@ -1688,7 +1704,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
                                   (void *)s, s->lineno, s->col_offset,
                                   s->end_lineno, s->end_col_offset))
             VISIT_QUIT(st, 0);
-        tmp = st->st_private;
         st->st_private = s->v.ClassDef.name;
         if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
             if (!symtable_add_def(st, &_Py_ID(__type_params__),
@@ -1702,13 +1717,13 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             }
         }
         VISIT_SEQ(st, stmt, s->v.ClassDef.body);
-        st->st_private = tmp;
         if (!symtable_exit_block(st))
             VISIT_QUIT(st, 0);
         if (asdl_seq_LEN(s->v.ClassDef.type_params) > 0) {
             if (!symtable_exit_block(st))
                 VISIT_QUIT(st, 0);
         }
+        st->st_private = tmp;
         break;
     }
     case TypeAlias_kind: {
@@ -2776,6 +2791,26 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
     return st;
 }
 
+PyObject *
+_Py_MaybeMangle(PyObject *privateobj, PySTEntryObject *ste, PyObject *name)
+{
+    /* Special case for type parameter blocks around generic classes:
+     * we want to mangle type parameter names (so a type param with a private
+     * name can be used inside the class body), but we don't want to mangle
+     * any other names that appear within the type parameter scope.
+     */
+    if (ste->ste_mangled_names != NULL) {
+        int result = PySet_Contains(ste->ste_mangled_names, name);
+        if (result < 0) {
+            return NULL;
+        }
+        if (result == 0) {
+            return Py_NewRef(name);
+        }
+    }
+    return _Py_Mangle(privateobj, name);
+}
+
 PyObject *
 _Py_Mangle(PyObject *privateobj, PyObject *ident)
 {

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Fix bug where names appearing after a generic class are mangled as if they`
	`2`	`+are in the generic class.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Fix bug where names are unexpectedly mangled in the bases of generic`
	`2`	`+classes.`
Original file line number	Diff line number	Diff line change
`@@ -1001,7 +1001,7 @@ static int`
`1001`	`1001`	`compiler_addop_name(struct compiler_unit *u, location loc,`
`1002`	`1002`	`int opcode, PyObject dict, PyObject o)`
`1003`	`1003`	`{`
`1004`		`- PyObject *mangled = _Py_Mangle(u->u_private, o);`
	`1004`	`+ PyObject *mangled = _Py_MaybeMangle(u->u_private, u->u_ste, o);`
`1005`	`1005`	`if (!mangled) {`
`1006`	`1006`	`return ERROR;`
`1007`	`1007`	`}`
`@@ -1873,7 +1873,7 @@ compiler_visit_kwonlydefaults(struct compiler *c, location loc,`
`1873`	`1873`	`arg_ty arg = asdl_seq_GET(kwonlyargs, i);`
`1874`	`1874`	`expr_ty default_ = asdl_seq_GET(kw_defaults, i);`
`1875`	`1875`	`if (default_) {`
`1876`		`- PyObject *mangled = _Py_Mangle(c->u->u_private, arg->arg);`
	`1876`	`+ PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, arg->arg);`
`1877`	`1877`	`if (!mangled) {`
`1878`	`1878`	`goto error;`
`1879`	`1879`	`}`
`@@ -1930,7 +1930,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id,`
`1930`	`1930`	`if (!annotation) {`
`1931`	`1931`	`return SUCCESS;`
`1932`	`1932`	`}`
`1933`		`- PyObject *mangled = _Py_Mangle(c->u->u_private, id);`
	`1933`	`+ PyObject *mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, id);`
`1934`	`1934`	`if (!mangled) {`
`1935`	`1935`	`return ERROR;`
`1936`	`1936`	`}`
`@@ -2625,7 +2625,6 @@ compiler_class(struct compiler *c, stmt_ty s)`
`2625`	`2625`	`asdl_type_param_seq *type_params = s->v.ClassDef.type_params;`
`2626`	`2626`	`int is_generic = asdl_seq_LEN(type_params) > 0;`
`2627`	`2627`	`if (is_generic) {`
`2628`		`- Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));`
`2629`	`2628`	`PyObject *type_params_name = PyUnicode_FromFormat("<generic parameters of %U>",`
`2630`	`2629`	`s->v.ClassDef.name);`
`2631`	`2630`	`if (!type_params_name) {`
`@@ -2637,6 +2636,7 @@ compiler_class(struct compiler *c, stmt_ty s)`
`2637`	`2636`	`return ERROR;`
`2638`	`2637`	`}`
`2639`	`2638`	`Py_DECREF(type_params_name);`
	`2639`	`+ Py_XSETREF(c->u->u_private, Py_NewRef(s->v.ClassDef.name));`
`2640`	`2640`	`RETURN_IF_ERROR_IN_SCOPE(c, compiler_type_params(c, type_params));`
`2641`	`2641`	`_Py_DECLARE_STR(type_params, ".type_params");`
`2642`	`2642`	`RETURN_IF_ERROR_IN_SCOPE(c, compiler_nameop(c, loc, &_Py_STR(type_params), Store));`
`@@ -4203,7 +4203,7 @@ compiler_nameop(struct compiler *c, location loc,`
`4203`	`4203`	`return ERROR;`
`4204`	`4204`	`}`
`4205`	`4205`
`4206`		`- mangled = _Py_Mangle(c->u->u_private, name);`
	`4206`	`+ mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, name);`
`4207`	`4207`	`if (!mangled) {`
`4208`	`4208`	`return ERROR;`
`4209`	`4209`	`}`
`@@ -6512,7 +6512,7 @@ compiler_annassign(struct compiler *c, stmt_ty s)`
`6512`	`6512`	`VISIT(c, expr, s->v.AnnAssign.annotation);`
`6513`	`6513`	`}`
`6514`	`6514`	`ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names);`
`6515`		`- mangled = _Py_Mangle(c->u->u_private, targ->v.Name.id);`
	`6515`	`+ mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id);`
`6516`	`6516`	`ADDOP_LOAD_CONST_NEW(c, loc, mangled);`
`6517`	`6517`	`ADDOP(c, loc, STORE_SUBSCR);`
`6518`	`6518`	`}`