diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 45644d6c092782..1e1e9e32dc2f76 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -750,6 +750,32 @@ def iter_raises(): self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], expected) + def test_freevar_through_scope_containing_comprehension(self): + code = """ + x = 1 + def f(): + for z in [x for x in [2]]: + pass + def g(): + return x + return g(), z + y, z = f() + """ + self._check_in_scopes(code, {"x": 1, "y": 1, "z": 2}, scopes=["module", "function"]) + + def test_freevar_through_scope_containing_comprehension_with_cell(self): + code = """ + x = 1 + def f(): + for l in [lambda: x for x in [2]]: + z = l() + def g(): + return x + return g(), z + y, z = f() + """ + self._check_in_scopes(code, {"x": 1, "y": 1, "z": 2}, scopes=["function"]) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst new file mode 100644 index 00000000000000..24e59bc48b2f19 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-27-20-28-27.gh-issue-121377.Svz8LR.rst @@ -0,0 +1 @@ +Fix multi-level non-local reference with intervening inlined comprehension binding the same name. diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 69d7e0a872aa48..eb037b54aacc0d 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -2700,7 +2700,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl } cfg_instr make_cell = { .i_opcode = MAKE_CELL, - // This will get fixed in offset_derefs(). + // This will get fixed in fix_cell_offsets(). .i_oparg = oldindex, .i_loc = NO_LOCATION, .i_target = NULL, diff --git a/Python/symtable.c b/Python/symtable.c index 8bc9db6d7d6811..428b267b338716 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -793,7 +793,7 @@ is_free_in_any_child(PySTEntryObject *entry, PyObject *key) static int inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *scopes, PyObject *comp_free, - PyObject *inlined_cells) + PyObject *inlined_cells, PyObject *inlined_locals) { PyObject *k, *v; Py_ssize_t pos = 0; @@ -843,6 +843,11 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, return 0; } SET_SCOPE(scopes, k, scope); + if (scope == LOCAL || scope == CELL) { + if (PySet_Add(inlined_locals, k) < 0) { + return 0; + } + } } else { long flags = PyLong_AsLong(existing); @@ -882,21 +887,27 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, */ static int -analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) +analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells, PyObject *inlined_locals) { - PyObject *name, *v, *v_cell; + PyObject *name, *v, *v_cell, *v_free; int success = 0; Py_ssize_t pos = 0; v_cell = PyLong_FromLong(CELL); if (!v_cell) return 0; + v_free = PyLong_FromLong(FREE); + if (!v_free) { + Py_DECREF(v_cell); + return 0; + } while (PyDict_Next(scopes, &pos, &name, &v)) { long scope = PyLong_AsLong(v); if (scope == -1 && PyErr_Occurred()) { goto error; } - if (scope != LOCAL) + + if (scope != LOCAL && scope != CELL) continue; int contains = PySet_Contains(free, name); if (contains < 0) { @@ -910,7 +921,21 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) if (!contains) { continue; } + } else { + /* If a free name is defined only by an inlined comprehension, it must be + * preserved as free in the outer scope. */ + contains = PySet_Contains(inlined_locals, name); + if (contains < 0) { + goto error; + } + if (contains) { + if (PyDict_SetItem(scopes, name, v_free) < 0) { + goto error; + } + continue; + } } + /* Replace LOCAL with CELL for this name, and remove from free. It is safe to replace the value of name in the dict, because it will not cause a resize. @@ -923,6 +948,7 @@ analyze_cells(PyObject *scopes, PyObject *free, PyObject *inlined_cells) success = 1; error: Py_DECREF(v_cell); + Py_DECREF(v_free); return success; } @@ -1099,7 +1125,8 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, PySTEntryObject *class_entry) { PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL; - PyObject *newglobal = NULL, *newfree = NULL, *inlined_cells = NULL; + PyObject *newglobal = NULL, *newfree = NULL; + PyObject *inlined_cells = NULL, *inlined_locals = NULL; PyObject *temp; int success = 0; Py_ssize_t i, pos = 0; @@ -1134,6 +1161,9 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, inlined_cells = PySet_New(NULL); if (!inlined_cells) goto error; + inlined_locals = PySet_New(NULL); + if (!inlined_locals) + goto error; /* Class namespace has no effect on names visible in nested functions, so populate the global and bound @@ -1231,7 +1261,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, goto error; } if (inline_comp) { - if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells)) { + if (!inline_comprehension(ste, entry, scopes, child_free, inlined_cells, inlined_locals)) { Py_DECREF(child_free); goto error; } @@ -1259,7 +1289,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, } /* Check if any local variables must be converted to cell variables */ - if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells)) + if (_PyST_IsFunctionLike(ste) && !analyze_cells(scopes, newfree, inlined_cells, inlined_locals)) goto error; else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree)) goto error; @@ -1280,6 +1310,7 @@ analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free, Py_XDECREF(newglobal); Py_XDECREF(newfree); Py_XDECREF(inlined_cells); + Py_XDECREF(inlined_locals); if (!success) assert(PyErr_Occurred()); return success;