gh-105858: Improve AST node constructors

Doc/library/ast.rst

@@ -103,20 +103,15 @@ Node classes
  For example, to create and populate an :class:`ast.UnaryOp` node, you could
  use ::
 
- node = ast.UnaryOp()
- node.op = ast.USub()
- node.operand = ast.Constant()
- node.operand.value = 5
- node.operand.lineno = 0
- node.operand.col_offset = 0
- node.lineno = 0
- node.col_offset = 0
-
- or the more compact ::
-
  node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0),
  lineno=0, col_offset=0)
 
+ If a field that is optional in the grammar is omitted from the constructor,
+ it defaults to ``None``. If a list field is omitted, it defaults to the empty
+ list. If any other field is omitted, a :exc:`DeprecationWarning` is raised
+ and the AST node will not have this field. In Python 3.15, this condition will
+ raise an error.
+
 .. versionchanged:: 3.8
 
  Class :class:`ast.Constant` is now used for all constants.
@@ -140,6 +135,14 @@ Node classes
  In the meantime, instantiating them will return an instance of
  a different class.
 
+.. deprecated-removed:: 3.13 3.15
+
+ Previous versions of Python allowed the creation of AST nodes that were missing
+ required fields. Similarly, AST node constructors allowed arbitrary keyword
+ arguments that were set as attributes of the AST node, even if they did not
+ match any of the fields of the AST node. This behavior is deprecated and will
+ be removed in Python 3.15.
+
 .. note::
  The descriptions of the specific node classes displayed here
  were initially adapted from the fantastic `Green Tree

Doc/whatsnew/3.13.rst

@@ -206,6 +206,21 @@ array
 ast
 ---
 
+* The constructors of node types in the :mod:`ast` module are now stricter
+ in the arguments they accept, and have more intuitive behaviour when
+ arguments are omitted.
+
+ If an optional field on an AST node is not included as an argument when
+ constructing an instance, the field will now be set to ``None``. Similarly,
+ if a list field is omitted, that field will now be set to an empty list.
+ (Previously, in both cases, the attribute would be missing on the newly
+ constructed AST node instance.)
+
+ If other arguments are omitted, a :exc:`DeprecationWarning` is emitted.
+ This will cause an exception in Python 3.15. Similarly, passing a keyword
+ argument that does not map to a field on the AST node is now deprecated,
+ and will raise an exception in Python 3.15.
+
 * :func:`ast.parse` now accepts an optional argument ``optimize``
  which is passed on to the :func:`compile` built-in. This makes it
  possible to obtain an optimized ``AST``.

Include/internal/pycore_global_strings.h

@@ -242,6 +242,7 @@ struct _Py_global_strings {
  STRUCT_FOR_ID(_check_retval_)
  STRUCT_FOR_ID(_dealloc_warn)
  STRUCT_FOR_ID(_feature_version)
+ STRUCT_FOR_ID(_field_types)
  STRUCT_FOR_ID(_fields_)
  STRUCT_FOR_ID(_finalizing)
  STRUCT_FOR_ID(_find_and_load)

Lib/test/test_ast.py

@@ -525,17 +525,38 @@ def test_field_attr_existence(self):
  if name == 'Index':
  continue
  if self._is_ast_node(name, item):
- x = item()
+ x = self._construct_ast_class(item)
  if isinstance(x, ast.AST):
  self.assertIs(type(x._fields), tuple)
 
+ def _construct_ast_class(self, cls):
+ kwargs = {}
+ for name, typ in cls.__annotations__.items():
+ if typ is str:
+ kwargs[name] = 'capybara'
+ elif typ is int:
+ kwargs[name] = 42
+ elif typ is object:
+ kwargs[name] = b'capybara'
+ elif isinstance(typ, type) and issubclass(typ, ast.AST):
+ kwargs[name] = self._construct_ast_class(typ)
+ return cls(**kwargs)
+
  def test_arguments(self):
  x = ast.arguments()
  self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs',
  'kw_defaults', 'kwarg', 'defaults'))
-
- with self.assertRaises(AttributeError):
- x.args
+ self.assertEqual(x.__annotations__, {
+ 'posonlyargs': list[ast.arg],
+ 'args': list[ast.arg],
+ 'vararg': ast.arg | None,
+ 'kwonlyargs': list[ast.arg],
+ 'kw_defaults': list[ast.expr],
+ 'kwarg': ast.arg | None,
+ 'defaults': list[ast.expr],
+ })
+
+ self.assertEqual(x.args, [])
  self.assertIsNone(x.vararg)
 
  x = ast.arguments(*range(1, 8))
@@ -551,7 +572,7 @@ def test_field_attr_writable_deprecated(self):
  self.assertEqual(x._fields, 666)
 
  def test_field_attr_writable(self):
- x = ast.Constant()
+ x = ast.Constant(1)
  # We can assign to _fields
  x._fields = 666
  self.assertEqual(x._fields, 666)
@@ -611,15 +632,22 @@ def test_classattrs_deprecated(self):
 
  self.assertEqual([str(w.message) for w in wlog], [
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
+ "Constant.__init__ missing 1 required positional argument: 'value'. This will become "
+ 'an error in Python 3.15.',
  'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
  'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
+ "Constant.__init__ missing 1 required positional argument: 'value'. This will become "
+ 'an error in Python 3.15.',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
  'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
+ "Constant.__init__ got an unexpected keyword argument 'foo'. Support for "
+ 'arbitrary keyword arguments is deprecated and will be removed in Python '
+ '3.15.',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
  'Attribute n is deprecated and will be removed in Python 3.14; use value instead',
  'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead',
@@ -636,7 +664,8 @@ def test_classattrs_deprecated(self):
  ])
 
  def test_classattrs(self):
- x = ast.Constant()
+ with self.assertWarns(DeprecationWarning):
+ x = ast.Constant()
  self.assertEqual(x._fields, ('value', 'kind'))
 
  with self.assertRaises(AttributeError):
@@ -651,7 +680,7 @@ def test_classattrs(self):
  with self.assertRaises(AttributeError):
  x.foobar
 
- x = ast.Constant(lineno=2)
+ x = ast.Constant(lineno=2, value=3)
  self.assertEqual(x.lineno, 2)
 
  x = ast.Constant(42, lineno=0)
@@ -662,8 +691,9 @@ def test_classattrs(self):
  self.assertRaises(TypeError, ast.Constant, 1, None, 2)
  self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0)
 
- # Arbitrary keyword arguments are supported
- self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar')
+ # Arbitrary keyword arguments are supported (but deprecated)
+ with self.assertWarns(DeprecationWarning):
+ self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar')
 
  with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"):
  ast.Constant(1, value=2)
@@ -815,11 +845,11 @@ def test_isinstance(self):
  assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes)
  assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant)
  assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis)
- assertNumDeprecated(self.assertNotIsInstance, Constant(), Num)
- assertStrDeprecated(self.assertNotIsInstance, Constant(), Str)
- assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes)
- assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant)
- assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis)
+ assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num)
+ assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str)
+ assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes)
+ assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant)
+ assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis)
 
  class S(str): pass
  with assertStrDeprecated():
@@ -888,8 +918,9 @@ def test_module(self):
  self.assertEqual(x.body, body)
 
  def test_nodeclasses(self):
- # Zero arguments constructor explicitly allowed
- x = ast.BinOp()
+ # Zero arguments constructor explicitly allowed (but deprecated)
+ with self.assertWarns(DeprecationWarning):
+ x = ast.BinOp()
  self.assertEqual(x._fields, ('left', 'op', 'right'))
 
  # Random attribute allowed too
@@ -927,8 +958,9 @@ def test_nodeclasses(self):
  self.assertEqual(x.right, 3)
  self.assertEqual(x.lineno, 0)
 
- # Random kwargs also allowed
- x = ast.BinOp(1, 2, 3, foobarbaz=42)
+ # Random kwargs also allowed (but deprecated)
+ with self.assertWarns(DeprecationWarning):
+ x = ast.BinOp(1, 2, 3, foobarbaz=42)
  self.assertEqual(x.foobarbaz, 42)
 
  def test_no_fields(self):
@@ -941,8 +973,9 @@ def test_pickling(self):
 
  for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
  for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests):
- ast2 = pickle.loads(pickle.dumps(ast, protocol))
- self.assertEqual(to_tuple(ast2), to_tuple(ast))
+ with self.subTest(ast=ast, protocol=protocol):
+ ast2 = pickle.loads(pickle.dumps(ast, protocol))
+ self.assertEqual(to_tuple(ast2), to_tuple(ast))
 
  def test_invalid_sum(self):
  pos = dict(lineno=2, col_offset=3)
@@ -1310,8 +1343,9 @@ def test_copy_location(self):
  'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, '
  'col_offset=0, end_lineno=1, end_col_offset=5))'
  )
- src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1)
- new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None))
+ func = ast.Name('spam', ast.Load())
+ src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func)
+ new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func))
  self.assertIsNone(new.end_lineno)
  self.assertIsNone(new.end_col_offset)
  self.assertEqual(new.lineno, 1)
@@ -1570,15 +1604,15 @@ def test_level_as_none(self):
  self.assertIn('sleep', ns)
 
  def test_recursion_direct(self):
- e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+ e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
  e.operand = e
  with self.assertRaises(RecursionError):
  with support.infinite_recursion():
  compile(ast.Expression(e), "<test>", "eval")
 
  def test_recursion_indirect(self):
- e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
- f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0)
+ e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
+ f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1))
  e.operand = f
  f.operand = e
  with self.assertRaises(RecursionError):
@@ -2866,6 +2900,23 @@ def visit_Call(self, node: ast.Call):
  self.assertASTTransformation(PrintToLog, code, expected)
 
 
+class ASTConstructorTests(unittest.TestCase):
+ """Test the autogenerated constructors for AST nodes."""
+
+ def test_FunctionDef(self):
+ args = ast.arguments()
+ self.assertEqual(args.args, [])
+ self.assertEqual(args.posonlyargs, [])
+ with self.assertWarnsRegex(DeprecationWarning,
+ r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"):
+ node = ast.FunctionDef(args=args)
+ self.assertFalse(hasattr(node, "name"))
+ self.assertEqual(node.decorator_list, [])
+ node = ast.FunctionDef(name='foo', args=args)
+ self.assertEqual(node.name, 'foo')
+ self.assertEqual(node.decorator_list, [])
+
+
 @support.cpython_only
 class ModuleStateTests(unittest.TestCase):
  # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state.

Misc/NEWS.d/next/Core and Builtins/2023-06-16-21-29-06.gh-issue-105858.Q7h0EV.rst

@@ -0,0 +1,8 @@
+Improve the constructors for :mod:`ast` nodes. Arguments of list types now
+default to an empty list if omitted, and optional fields default to ``None``.
+AST nodes now have an
+``__annotations__`` attribute with the expected types of their attributes.
+Passing unrecognized extra arguments to AST nodes is deprecated and will
+become an error in Python 3.15. Omitting a required argument to an AST node
+is deprecated and will become an error in Python 3.15. Patch by Jelle
+Zijlstra.