diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 319b2c81505f48..210fa584269eb4 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -43,7 +43,7 @@ Node classes .. class:: AST - This is the base of all AST node classes. The actual node classes are + This is the abstract base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C module and re-exported in :mod:`ast`. @@ -161,6 +161,15 @@ Node classes match any of the fields of the AST node. This behavior is deprecated and will be removed in Python 3.15. +.. deprecated-removed:: next 3.20 + + In the :ref:`grammar above `, the AST node classes that + correspond to production rules with variants (aka "sums") are abstract + classes. Previous versions of Python allowed for the creation of direct + instances of these abstract node classes. This behavior is deprecated and + will be removed in Python 3.20. + + .. note:: The descriptions of the specific node classes displayed here were initially adapted from the fantastic `Green Tree diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 252d8966b7450f..1e0ee6c043ba9e 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -408,6 +408,14 @@ module_name Deprecated ========== +ast +--- + +* Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + (Contributed by Brian Schubert in :gh:`116021`.) + + hashlib ------- diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index d4ac419f51d6b2..9627b15bd69c5b 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -161,6 +161,7 @@ struct ast_state { PyObject *__module__; PyObject *_attributes; PyObject *_fields; + PyObject *abstract_types; PyObject *alias_type; PyObject *annotation; PyObject *arg; diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 1e6f60074308e2..38e8a279109194 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -84,7 +84,9 @@ def _assertTrueorder(self, ast_node, parent_pos): self.assertEqual(ast_node._fields, ast_node.__match_args__) def test_AST_objects(self): - x = ast.AST() + # Directly instantiating abstract node class AST is allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.AST() self.assertEqual(x._fields, ()) x.foobar = 42 self.assertEqual(x.foobar, 42) @@ -93,7 +95,7 @@ def test_AST_objects(self): with self.assertRaises(AttributeError): x.vararg - with self.assertRaises(TypeError): + with self.assertRaises(TypeError), self.assertWarns(DeprecationWarning): # "ast.AST constructor takes 0 positional arguments" ast.AST(2) @@ -109,15 +111,15 @@ def cleanup(): msg = "type object 'ast.AST' has no attribute '_fields'" # Both examples used to crash: - with self.assertRaisesRegex(AttributeError, msg): + with self.assertRaisesRegex(AttributeError, msg), self.assertWarns(DeprecationWarning): ast.AST(arg1=123) - with self.assertRaisesRegex(AttributeError, msg): + with self.assertRaisesRegex(AttributeError, msg), self.assertWarns(DeprecationWarning): ast.AST() - def test_AST_garbage_collection(self): + def test_node_garbage_collection(self): class X: pass - a = ast.AST() + a = ast.Module() a.x = X() a.x.a = a ref = weakref.ref(a.x) @@ -427,7 +429,12 @@ def _construct_ast_class(self, cls): elif typ is object: kwargs[name] = b'capybara' elif isinstance(typ, type) and issubclass(typ, ast.AST): - kwargs[name] = self._construct_ast_class(typ) + if typ._is_abstract(): + # Use an arbitrary concrete subclass + concrete = next(sub for sub in typ.__subclasses__() if not sub._is_abstract()) + kwargs[name] = self._construct_ast_class(concrete) + else: + kwargs[name] = self._construct_ast_class(typ) return cls(**kwargs) def test_arguments(self): @@ -573,6 +580,10 @@ def test_nodeclasses(self): x = ast.BinOp(1, 2, 3, foobarbaz=42) self.assertEqual(x.foobarbaz, 42) + # Directly instantiating abstract node types is allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + ast.stmt() + def test_no_fields(self): # this used to fail because Sub._fields was None x = ast.Sub() @@ -580,7 +591,9 @@ def test_no_fields(self): def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) + with self.assertWarns(DeprecationWarning): + # Creating instances of ast.expr is deprecated + m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) with self.assertRaises(TypeError) as cm: compile(m, "", "exec") self.assertIn("but got expr()", str(cm.exception)) @@ -1140,6 +1153,9 @@ def do(cls): return if cls is ast.Index: return + # Don't attempt to create instances of abstract AST nodes + if cls._is_abstract(): + return yield cls for sub in cls.__subclasses__(): diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index f89237931b7185..bbf4f20107bdcf 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1877,7 +1877,8 @@ def test_pythontypes(self): check = self.check_sizeof # _ast.AST import _ast - check(_ast.AST(), size('P')) + with self.assertWarns(DeprecationWarning): + check(_ast.AST(), size('P')) try: raise TypeError except TypeError as e: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst new file mode 100644 index 00000000000000..967d8faaef3422 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-16-12-56-08.gh-issue-116021.hMN9yw.rst @@ -0,0 +1,2 @@ +Support for creating instances of abstract AST nodes from the :mod:`ast` module +is deprecated and scheduled for removal in Python 3.20. Patch by Brian Schubert. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index dba20226c3283a..2b28f18ea4835f 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -881,6 +881,21 @@ def visitModule(self, mod): return -1; } + int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self)); + if (contains == -1) { + return -1; + } + else if (contains == 1) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Instantiating abstract AST node class %T is deprecated. " + "This will become an error in Python 3.20", + self + ) < 0) { + return -1; + } + } + Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; @@ -1443,6 +1458,23 @@ def visitModule(self, mod): return result; } +/* Helper for checking if a node class is abstract in the tests. */ +static PyObject * +ast_is_abstract(PyObject *cls, void *Py_UNUSED(ignored)) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + int contains = PySet_Contains(state->abstract_types, cls); + if (contains == -1) { + return NULL; + } + else if (contains == 1) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyMemberDef ast_type_members[] = { {"__dictoffset__", Py_T_PYSSIZET, offsetof(AST_object, dict), Py_READONLY}, {NULL} /* Sentinel */ @@ -1454,6 +1486,7 @@ def visitModule(self, mod): PyDoc_STR("__replace__($self, /, **fields)\\n--\\n\\n" "Return a copy of the AST node with new values " "for the specified fields.")}, + {"_is_abstract", _PyCFunction_CAST(ast_is_abstract), METH_CLASS | METH_NOARGS, NULL}, {NULL} }; @@ -1887,6 +1920,13 @@ def visitModule(self, mod): if (!state->AST_type) { return -1; } + state->abstract_types = PySet_New(NULL); + if (!state->abstract_types) { + return -1; + } + if (PySet_Add(state->abstract_types, state->AST_type) < 0) { + return -1; + } if (add_ast_fields(state) < 0) { return -1; } @@ -1928,6 +1968,7 @@ def visitSum(self, sum, name): (name, name, len(sum.attributes)), 1) else: self.emit("if (add_attributes(state, state->%s_type, NULL, 0) < 0) return -1;" % name, 1) + self.emit("if (PySet_Add(state->abstract_types, state->%s_type) < 0) return -1;" % name, 1) self.emit_defaults(name, sum.attributes, 1) simple = is_simple(sum) for t in sum.types: @@ -2289,6 +2330,7 @@ def generate_module_def(mod, metadata, f, internal_h): "%s_type" % type for type in metadata.types ) + module_state.add("abstract_types") state_strings = sorted(state_strings) module_state = sorted(module_state) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 660bc598a4862c..3bf58d994a74fa 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -178,6 +178,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->__module__); Py_CLEAR(state->_attributes); Py_CLEAR(state->_fields); + Py_CLEAR(state->abstract_types); Py_CLEAR(state->alias_type); Py_CLEAR(state->annotation); Py_CLEAR(state->arg); @@ -5165,6 +5166,21 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) return -1; } + int contains = PySet_Contains(state->abstract_types, (PyObject *)Py_TYPE(self)); + if (contains == -1) { + return -1; + } + else if (contains == 1) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Instantiating abstract AST node class %T is deprecated. " + "This will become an error in Python 3.20", + self + ) < 0) { + return -1; + } + } + Py_ssize_t i, numfields = 0; int res = -1; PyObject *key, *value, *fields, *attributes = NULL, *remaining_fields = NULL; @@ -5727,6 +5743,23 @@ ast_type_replace(PyObject *self, PyObject *args, PyObject *kwargs) return result; } +/* Helper for checking if a node class is abstract in the tests. */ +static PyObject * +ast_is_abstract(PyObject *cls, void *Py_UNUSED(ignored)) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + int contains = PySet_Contains(state->abstract_types, cls); + if (contains == -1) { + return NULL; + } + else if (contains == 1) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyMemberDef ast_type_members[] = { {"__dictoffset__", Py_T_PYSSIZET, offsetof(AST_object, dict), Py_READONLY}, {NULL} /* Sentinel */ @@ -5738,6 +5771,7 @@ static PyMethodDef ast_type_methods[] = { PyDoc_STR("__replace__($self, /, **fields)\n--\n\n" "Return a copy of the AST node with new values " "for the specified fields.")}, + {"_is_abstract", _PyCFunction_CAST(ast_is_abstract), METH_CLASS | METH_NOARGS, NULL}, {NULL} }; @@ -6170,6 +6204,13 @@ init_types(void *arg) if (!state->AST_type) { return -1; } + state->abstract_types = PySet_New(NULL); + if (!state->abstract_types) { + return -1; + } + if (PySet_Add(state->abstract_types, state->AST_type) < 0) { + return -1; + } if (add_ast_fields(state) < 0) { return -1; } @@ -6180,6 +6221,7 @@ init_types(void *arg) " | FunctionType(expr* argtypes, expr returns)"); if (!state->mod_type) return -1; if (add_attributes(state, state->mod_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->mod_type) < 0) return -1; state->Module_type = make_type(state, "Module", state->mod_type, Module_fields, 2, "Module(stmt* body, type_ignore* type_ignores)"); @@ -6229,6 +6271,7 @@ init_types(void *arg) if (!state->stmt_type) return -1; if (add_attributes(state, state->stmt_type, stmt_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->stmt_type) < 0) return -1; if (PyObject_SetAttr(state->stmt_type, state->end_lineno, Py_None) == -1) return -1; if (PyObject_SetAttr(state->stmt_type, state->end_col_offset, Py_None) == @@ -6414,6 +6457,7 @@ init_types(void *arg) if (!state->expr_type) return -1; if (add_attributes(state, state->expr_type, expr_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->expr_type) < 0) return -1; if (PyObject_SetAttr(state->expr_type, state->end_lineno, Py_None) == -1) return -1; if (PyObject_SetAttr(state->expr_type, state->end_col_offset, Py_None) == @@ -6558,6 +6602,8 @@ init_types(void *arg) "expr_context = Load | Store | Del"); if (!state->expr_context_type) return -1; if (add_attributes(state, state->expr_context_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->expr_context_type) < 0) return + -1; state->Load_type = make_type(state, "Load", state->expr_context_type, NULL, 0, "Load"); @@ -6582,6 +6628,7 @@ init_types(void *arg) "boolop = And | Or"); if (!state->boolop_type) return -1; if (add_attributes(state, state->boolop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->boolop_type) < 0) return -1; state->And_type = make_type(state, "And", state->boolop_type, NULL, 0, "And"); if (!state->And_type) return -1; @@ -6599,6 +6646,7 @@ init_types(void *arg) "operator = Add | Sub | Mult | MatMult | Div | Mod | Pow | LShift | RShift | BitOr | BitXor | BitAnd | FloorDiv"); if (!state->operator_type) return -1; if (add_attributes(state, state->operator_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->operator_type) < 0) return -1; state->Add_type = make_type(state, "Add", state->operator_type, NULL, 0, "Add"); if (!state->Add_type) return -1; @@ -6693,6 +6741,7 @@ init_types(void *arg) "unaryop = Invert | Not | UAdd | USub"); if (!state->unaryop_type) return -1; if (add_attributes(state, state->unaryop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->unaryop_type) < 0) return -1; state->Invert_type = make_type(state, "Invert", state->unaryop_type, NULL, 0, "Invert"); @@ -6723,6 +6772,7 @@ init_types(void *arg) "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn"); if (!state->cmpop_type) return -1; if (add_attributes(state, state->cmpop_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->cmpop_type) < 0) return -1; state->Eq_type = make_type(state, "Eq", state->cmpop_type, NULL, 0, "Eq"); if (!state->Eq_type) return -1; @@ -6796,6 +6846,8 @@ init_types(void *arg) if (!state->excepthandler_type) return -1; if (add_attributes(state, state->excepthandler_type, excepthandler_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->excepthandler_type) < 0) return + -1; if (PyObject_SetAttr(state->excepthandler_type, state->end_lineno, Py_None) == -1) return -1; @@ -6886,6 +6938,7 @@ init_types(void *arg) if (!state->pattern_type) return -1; if (add_attributes(state, state->pattern_type, pattern_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->pattern_type) < 0) return -1; state->MatchValue_type = make_type(state, "MatchValue", state->pattern_type, MatchValue_fields, 1, @@ -6936,6 +6989,8 @@ init_types(void *arg) "type_ignore = TypeIgnore(int lineno, string tag)"); if (!state->type_ignore_type) return -1; if (add_attributes(state, state->type_ignore_type, NULL, 0) < 0) return -1; + if (PySet_Add(state->abstract_types, state->type_ignore_type) < 0) return + -1; state->TypeIgnore_type = make_type(state, "TypeIgnore", state->type_ignore_type, TypeIgnore_fields, 2, @@ -6949,6 +7004,7 @@ init_types(void *arg) if (!state->type_param_type) return -1; if (add_attributes(state, state->type_param_type, type_param_attributes, 4) < 0) return -1; + if (PySet_Add(state->abstract_types, state->type_param_type) < 0) return -1; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, TypeVar_fields, 3, "TypeVar(identifier name, expr? bound, expr? default_value)");