diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 25f9cba390de68..54ef70d512dcf6 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -1,6 +1,11 @@ Pending removal in Python 3.19 ------------------------------ +* :mod:`ast`: + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.19. + * :mod:`ctypes`: * Implicitly switching to the MSVC-compatible struct layout by setting diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 319b2c81505f48..9d3a83d25cd6d9 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -43,7 +43,7 @@ Node classes .. class:: AST - This is the base of all AST node classes. The actual node classes are + This is the abstract base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C module and re-exported in :mod:`ast`. @@ -161,6 +161,15 @@ Node classes match any of the fields of the AST node. This behavior is deprecated and will be removed in Python 3.15. +.. deprecated-removed:: next 3.19 + + In the :ref:`grammar above `, the AST node classes that + correspond to production rules with variants (aka "sums") are abstract + classes. Previous versions of Python allowed for the creation of direct + instances of these abstract node classes. This behavior is deprecated and + will be removed in Python 3.19. + + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 9031301989e396..d26b5902a7ef6e 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -537,6 +537,14 @@ module_name Deprecated ========== +ast +--- + +* Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.19. + (Contributed by Brian Schubert in :gh:`116021`.) + + CLI --- @@ -548,6 +556,7 @@ CLI (Contributed by Nikita Sobolev in :gh:`136355`.) + hashlib ------- diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index d4ac419f51d6b2..9627b15bd69c5b 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -161,6 +161,7 @@ struct ast_state { PyObject *__module__; PyObject *_attributes; PyObject *_fields; + PyObject *abstract_types; PyObject *alias_type; PyObject *annotation; PyObject *arg; diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 1e6f60074308e2..6aa9c1899cc9ba 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -1,4 +1,5 @@ import _ast_unparse +import _ast import ast import builtins import contextlib @@ -84,7 +85,9 @@ def _assertTrueorder(self, ast_node, parent_pos): self.assertEqual(ast_node._fields, ast_node.__match_args__) def test_AST_objects(self): - x = ast.AST() + # Directly instantiating abstract node class AST is allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.AST() self.assertEqual(x._fields, ()) x.foobar = 42 self.assertEqual(x.foobar, 42) @@ -93,7 +96,7 @@ def test_AST_objects(self): with self.assertRaises(AttributeError): x.vararg - with self.assertRaises(TypeError): + with self.assertRaises(TypeError), self.assertWarns(DeprecationWarning): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) @@ -109,15 +112,21 @@ def cleanup(): msg = "type object 'ast.AST' has no attribute '_fields'" # Both examples used to crash: - with self.assertRaisesRegex(AttributeError, msg): + with ( + self.assertRaisesRegex(AttributeError, msg), + self.assertWarns(DeprecationWarning), + ): ast.AST(arg1=123) - with self.assertRaisesRegex(AttributeError, msg): + with ( + self.assertRaisesRegex(AttributeError, msg), + self.assertWarns(DeprecationWarning), + ): ast.AST() - def test_AST_garbage_collection(self): + def test_node_garbage_collection(self): class X: pass - a = ast.AST() + a = ast.Module() a.x = X() a.x.a = a ref = weakref.ref(a.x) @@ -427,7 +436,15 @@ def _construct_ast_class(self, cls): elif typ is object: kwargs[name] = b'capybara' elif isinstance(typ, type) and issubclass(typ, ast.AST): - kwargs[name] = self._construct_ast_class(typ) + if _ast._is_abstract(typ): + # Use an arbitrary concrete subclass + concrete = next((sub for sub in typ.__subclasses__() + if not _ast._is_abstract(sub)), None) + msg = f"abstract node class {typ} has no concrete subclasses" + self.assertIsNotNone(concrete, msg) + else: + concrete = typ + kwargs[name] = self._construct_ast_class(concrete) return cls(**kwargs) def test_arguments(self): @@ -573,6 +590,10 @@ def test_nodeclasses(self): x = ast.BinOp(1, 2, 3, foobarbaz=42) self.assertEqual(x.foobarbaz, 42) + # Directly instantiating abstract node types is allowed (but deprecated) + self.assertWarns(DeprecationWarning, ast.stmt) + self.assertWarns(DeprecationWarning, ast.expr_context) + def test_no_fields(self): # this used to fail because Sub._fields was None x = ast.Sub() @@ -580,7 +601,10 @@ def test_no_fields(self): def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) + with self.assertWarns(DeprecationWarning): + # Creating instances of ast.expr is deprecated + e = ast.expr(**pos) + m = ast.Module([ast.Expr(e, **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") self.assertIn("but got expr()", str(cm.exception)) @@ -1132,14 +1156,19 @@ class CopyTests(unittest.TestCase): def iter_ast_classes(): """Iterate over the (native) subclasses of ast.AST recursively. - This excludes the special class ast.Index since its constructor - returns an integer. + This excludes: + * abstract AST nodes + * the special class ast.Index, since its constructor returns + an integer. """ def do(cls): if cls.__module__ != 'ast': return if cls is ast.Index: return + # Don't attempt to create instances of abstract AST nodes + if _ast._is_abstract(cls): + return yield cls for sub in cls.__subclasses__(): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1198c6d35113c8..e435f2449ad7d4 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1889,7 +1889,8 @@ def test_pythontypes(self): check = self.check_sizeof # _ast.AST import _ast - check(_ast.AST(), size('P')) + with self.assertWarns(DeprecationWarning): + check(_ast.AST(), size('P')) try: raise TypeError except TypeError as e: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst new file mode 100644 index 00000000000000..cfb9f974f244e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst @@ -0,0 +1,2 @@ +Support for creating instances of abstract AST nodes from the :mod:`ast` module +is deprecated and scheduled for removal in Python 3.19. Patch by Brian Schubert. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index dba20226c3283a..8807a5a7f527a4 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -881,6 +881,20 @@ def visitModule(self, mod): return -1; } + int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self)); + if (contains == -1) { + return -1; + } + else if (contains == 1) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Instantiating abstract AST node class %T is deprecated. " + "This will become an error in Python 3.19", self) < 0) + { + return -1; + } + } + Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; @@ -1887,6 +1901,13 @@ def visitModule(self, mod): if (!state->AST_type) { return -1; } + state->abstract_types = PySet_New(NULL); + if (!state->abstract_types) { + return -1; + } + if (PySet_Add(state->abstract_types, state->AST_type) < 0) { + return -1; + } if (add_ast_fields(state) < 0) { return -1; } @@ -1928,6 +1949,7 @@ def visitSum(self, sum, name): (name, name, len(sum.attributes)), 1) else: self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1) + self.emit("if (PySet_Add(state->abstract_types, state->%s_type) < 0) return -1;" % name, 1) self.emit_defaults(name, sum.attributes, 1) simple = is_simple(sum) for t in sum.types: @@ -1960,6 +1982,30 @@ def emit_defaults(self, name, fields, depth): class ASTModuleVisitor(PickleVisitor): def visitModule(self, mod): + self.emit(""" +/* Helper for checking if a node class is abstract in the tests. */ +static PyObject * +ast_is_abstract(PyObject *Py_UNUSED(module), PyObject *cls) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + int contains = PySet_Contains(state->abstract_types, cls); + if (contains == -1) { + return NULL; + } + else if (contains == 1) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static struct PyMethodDef astmodule_methods[] = { + {"_is_abstract", ast_is_abstract, METH_O, NULL}, + {NULL} /* Sentinel */ +}; +""".strip(), 0, reflow=False) + self.emit("", 0) self.emit("static int", 0) self.emit("astmodule_exec(PyObject *m)", 0) self.emit("{", 0) @@ -2000,7 +2046,8 @@ def visitModule(self, mod): .m_name = "_ast", // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, - .m_slots = astmodule_slots, + .m_methods = astmodule_methods, + .m_slots = astmodule_slots }; PyMODINIT_FUNC @@ -2289,6 +2336,7 @@ def generate_module_def(mod, metadata, f, internal_h): "%s_type" % type for type in metadata.types ) + module_state.add("abstract_types") state_strings = sorted(state_strings) module_state = sorted(module_state) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 660bc598a4862c..4c327ec4f64a79 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -178,6 +178,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->__module__); Py_CLEAR(state->_attributes); Py_CLEAR(state->_fields); + Py_CLEAR(state->abstract_types); Py_CLEAR(state->alias_type); Py_CLEAR(state->annotation); Py_CLEAR(state->arg); @@ -5165,6 +5166,20 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) return -1; } + int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self)); + if (contains == -1) { + return -1; + } + else if (contains == 1) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Instantiating abstract AST node class %T is deprecated. " + "This will become an error in Python 3.19", self) < 0) + { + return -1; + } + } + Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; @@ -6170,6 +6185,13 @@ init_types(void *arg) if (!state->AST_type) { return -1; } + state->abstract_types = PySet_New(NULL); + if (!state->abstract_types) { + return -1; + } + if (PySet_Add(state->abstract_types, state->AST_type) < 0) { + return -1; + } if (add_ast_fields(state) < 0) { return -1; } @@ -6180,6 +6202,7 @@ init_types(void *arg) " | FunctionType(expr* argtypes, expr returns)"); if (!state->mod_type) return -1; if (add_attributes(state, state->mod_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->mod_type) < 0) return -1; state->Module_type = make_type(state, "Module", state->mod_type, Module_fields, 2, "Module(stmt* body, type_ignore* type_ignores)"); @@ -6229,6 +6252,7 @@ init_types(void *arg) if (!state->stmt_type) return -1; if (add_attributes(state, state->stmt_type, stmt_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->stmt_type) < 0) return -1; if (PyObject_SetAttr(state->stmt_type, state->end_lineno, Py_None) == -1) return -1; if (PyObject_SetAttr(state->stmt_type, state->end_col_offset, Py_None) == @@ -6414,6 +6438,7 @@ init_types(void *arg) if (!state->expr_type) return -1; if (add_attributes(state, state->expr_type, expr_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->expr_type) < 0) return -1; if (PyObject_SetAttr(state->expr_type, state->end_lineno, Py_None) == -1) return -1; if (PyObject_SetAttr(state->expr_type, state->end_col_offset, Py_None) == @@ -6558,6 +6583,8 @@ init_types(void *arg) "expr_context = Load | Store | Del"); if (!state->expr_context_type) return -1; if (add_attributes(state, state->expr_context_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->expr_context_type) < 0) return + -1; state->Load_type = make_type(state, "Load", state->expr_context_type, NULL, 0, "Load"); @@ -6582,6 +6609,7 @@ init_types(void *arg) "boolop = And | Or"); if (!state->boolop_type) return -1; if (add_attributes(state, state->boolop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->boolop_type) < 0) return -1; state->And_type = make_type(state, "And", state->boolop_type, NULL, 0, "And"); if (!state->And_type) return -1; @@ -6599,6 +6627,7 @@ init_types(void *arg) "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv"); if (!state->operator_type) return -1; if (add_attributes(state, state->operator_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->operator_type) < 0) return -1; state->Add_type = make_type(state, "Add", state->operator_type, NULL, 0, "Add"); if (!state->Add_type) return -1; @@ -6693,6 +6722,7 @@ init_types(void *arg) "unaryop = Invert | Not | UAdd | USub"); if (!state->unaryop_type) return -1; if (add_attributes(state, state->unaryop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->unaryop_type) < 0) return -1; state->Invert_type = make_type(state, "Invert", state->unaryop_type, NULL, 0, "Invert"); @@ -6723,6 +6753,7 @@ init_types(void *arg) "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn"); if (!state->cmpop_type) return -1; if (add_attributes(state, state->cmpop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->cmpop_type) < 0) return -1; state->Eq_type = make_type(state, "Eq", state->cmpop_type, NULL, 0, "Eq"); if (!state->Eq_type) return -1; @@ -6796,6 +6827,8 @@ init_types(void *arg) if (!state->excepthandler_type) return -1; if (add_attributes(state, state->excepthandler_type, excepthandler_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->excepthandler_type) < 0) return + -1; if (PyObject_SetAttr(state->excepthandler_type, state->end_lineno, Py_None) == -1) return -1; @@ -6886,6 +6919,7 @@ init_types(void *arg) if (!state->pattern_type) return -1; if (add_attributes(state, state->pattern_type, pattern_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->pattern_type) < 0) return -1; state->MatchValue_type = make_type(state, "MatchValue", state->pattern_type, MatchValue_fields, 1, @@ -6936,6 +6970,8 @@ init_types(void *arg) "type_ignore = TypeIgnore(int lineno, string tag)"); if (!state->type_ignore_type) return -1; if (add_attributes(state, state->type_ignore_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->type_ignore_type) < 0) return + -1; state->TypeIgnore_type = make_type(state, "TypeIgnore", state->type_ignore_type, TypeIgnore_fields, 2, @@ -6949,6 +6985,7 @@ init_types(void *arg) if (!state->type_param_type) return -1; if (add_attributes(state, state->type_param_type, type_param_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->type_param_type) < 0) return -1; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, TypeVar_fields, 3, "TypeVar(identifier name, expr? bound, expr? default_value)"); @@ -17977,6 +18014,28 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, } +/* Helper for checking if a node class is abstract in the tests. */ +static PyObject * +ast_is_abstract(PyObject *Py_UNUSED(module), PyObject *cls) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + int contains = PySet_Contains(state->abstract_types, cls); + if (contains == -1) { + return NULL; + } + else if (contains == 1) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + +static struct PyMethodDef astmodule_methods[] = { + {"_is_abstract", ast_is_abstract, METH_O, NULL}, + {NULL} /* Sentinel */ +}; + static int astmodule_exec(PyObject *m) { @@ -18402,7 +18461,8 @@ static struct PyModuleDef _astmodule = { .m_name = "_ast", // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, - .m_slots = astmodule_slots, + .m_methods = astmodule_methods, + .m_slots = astmodule_slots }; PyMODINIT_FUNC