gh-117486: Improve behavior for user-defined AST subclasses (#118212)

Now, such classes will no longer require changes in Python 3.13 in the normal case.
The test suite for robotframework passes with no DeprecationWarnings under this PR.

I also added a new DeprecationWarning for the case where `_field_types` exists
but is incomplete, since that seems likely to indicate a user mistake.

6 files changed, 94 insertions, 33 deletions

diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
index e954c38..02dc7c8 100644
--- a/Doc/library/ast.rst
+++ b/Doc/library/ast.rst
@@ -61,7 +61,7 @@ Node classes
 
    .. attribute:: _fields
 
-      Each concrete class has an attribute :attr:`_fields` which gives the names
+      Each concrete class has an attribute :attr:`!_fields` which gives the names
       of all child nodes.
 
       Each instance of a concrete class has one attribute for each child node,
@@ -74,6 +74,18 @@ Node classes
       as Python lists.  All possible attributes must be present and have valid
       values when compiling an AST with :func:`compile`.
 
+   .. attribute:: _field_types
+
+      The :attr:`!_field_types` attribute on each concrete class is a dictionary
+      mapping field names (as also listed in :attr:`_fields`) to their types.
+
+      .. doctest::
+
+           >>> ast.TypeVar._field_types
+           {'name': <class 'str'>, 'bound': ast.expr | None, 'default_value': ast.expr | None}
+
+      .. versionadded:: 3.13
+
    .. attribute:: lineno
                   col_offset
                   end_lineno
diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst
index 43934ba..c82d8bd 100644
--- a/Doc/whatsnew/3.13.rst
+++ b/Doc/whatsnew/3.13.rst
@@ -384,6 +384,12 @@ ast
   argument that does not map to a field on the AST node is now deprecated,
   and will raise an exception in Python 3.15.
 
+  These changes do not apply to user-defined subclasses of :class:`ast.AST`,
+  unless the class opts in to the new behavior by setting the attribute
+  :attr:`ast.AST._field_types`.
+
+  (Contributed by Jelle Zijlstra in :gh:`105858` and :gh:`117486`.)
+
 * :func:`ast.parse` now accepts an optional argument *optimize*
   which is passed on to the :func:`compile` built-in. This makes it
   possible to obtain an optimized AST.
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
index 6d05c8f..f6e22d4 100644
--- a/Lib/test/test_ast.py
+++ b/Lib/test/test_ast.py
@@ -3036,7 +3036,7 @@ class ASTConstructorTests(unittest.TestCase):
         self.assertEqual(node.name, 'foo')
         self.assertEqual(node.decorator_list, [])
 
-    def test_custom_subclass(self):
+    def test_custom_subclass_with_no_fields(self):
         class NoInit(ast.AST):
             pass
 
@@ -3044,17 +3044,17 @@ class ASTConstructorTests(unittest.TestCase):
         self.assertIsInstance(obj, NoInit)
         self.assertEqual(obj.__dict__, {})
 
+    def test_fields_but_no_field_types(self):
         class Fields(ast.AST):
             _fields = ('a',)
 
-        with self.assertWarnsRegex(DeprecationWarning,
-                                   r"Fields provides _fields but not _field_types."):
-            obj = Fields()
+        obj = Fields()
         with self.assertRaises(AttributeError):
             obj.a
         obj = Fields(a=1)
         self.assertEqual(obj.a, 1)
 
+    def test_fields_and_types(self):
         class FieldsAndTypes(ast.AST):
             _fields = ('a',)
             _field_types = {'a': int | None}
@@ -3065,6 +3065,7 @@ class ASTConstructorTests(unittest.TestCase):
         obj = FieldsAndTypes(a=1)
         self.assertEqual(obj.a, 1)
 
+    def test_fields_and_types_no_default(self):
         class FieldsAndTypesNoDefault(ast.AST):
             _fields = ('a',)
             _field_types = {'a': int}
@@ -3077,6 +3078,38 @@ class ASTConstructorTests(unittest.TestCase):
         obj = FieldsAndTypesNoDefault(a=1)
         self.assertEqual(obj.a, 1)
 
+    def test_incomplete_field_types(self):
+        class MoreFieldsThanTypes(ast.AST):
+            _fields = ('a', 'b')
+            _field_types = {'a': int | None}
+            a: int | None = None
+            b: int | None = None
+
+        with self.assertWarnsRegex(
+            DeprecationWarning,
+            r"Field 'b' is missing from MoreFieldsThanTypes\._field_types"
+        ):
+            obj = MoreFieldsThanTypes()
+        self.assertIs(obj.a, None)
+        self.assertIs(obj.b, None)
+
+        obj = MoreFieldsThanTypes(a=1, b=2)
+        self.assertEqual(obj.a, 1)
+        self.assertEqual(obj.b, 2)
+
+    def test_complete_field_types(self):
+        class _AllFieldTypes(ast.AST):
+            _fields = ('a', 'b')
+            _field_types = {'a': int | None, 'b': list[str]}
+            # This must be set explicitly
+            a: int | None = None
+            # This will add an implicit empty list default
+            b: list[str]
+
+        obj = _AllFieldTypes()
+        self.assertIs(obj.a, None)
+        self.assertEqual(obj.b, [])
+
 
 @support.cpython_only
 class ModuleStateTests(unittest.TestCase):
diff --git a/Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst b/Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst
new file mode 100644
index 0000000..f02d895
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst
@@ -0,0 +1,4 @@
+Improve the behavior of user-defined subclasses of :class:`ast.AST`. Such
+classes will now require no changes in the usual case to conform with the
+behavior changes of the :mod:`ast` module in Python 3.13. Patch by Jelle
+Zijlstra.
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 1f0be45..11d59fa 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -979,14 +979,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
             goto cleanup;
         }
         if (field_types == NULL) {
-            if (PyErr_WarnFormat(
-                PyExc_DeprecationWarning, 1,
-                "%.400s provides _fields but not _field_types. "
-                "This will become an error in Python 3.15.",
-                Py_TYPE(self)->tp_name
-            ) < 0) {
-                res = -1;
-            }
+            // Probably a user-defined subclass of AST that lacks _field_types.
+            // This will continue to work as it did before 3.13; i.e., attributes
+            // that are not passed in simply do not exist on the instance.
             goto cleanup;
         }
         remaining_list = PySequence_List(remaining_fields);
@@ -997,12 +992,21 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
             PyObject *name = PyList_GET_ITEM(remaining_list, i);
             PyObject *type = PyDict_GetItemWithError(field_types, name);
             if (!type) {
-                if (!PyErr_Occurred()) {
-                    PyErr_SetObject(PyExc_KeyError, name);
+                if (PyErr_Occurred()) {
+                    goto set_remaining_cleanup;
+                }
+                else {
+                    if (PyErr_WarnFormat(
+                        PyExc_DeprecationWarning, 1,
+                        "Field '%U' is missing from %.400s._field_types. "
+                        "This will become an error in Python 3.15.",
+                        name, Py_TYPE(self)->tp_name
+                    ) < 0) {
+                        goto set_remaining_cleanup;
+                    }
                 }
-                goto set_remaining_cleanup;
             }
-            if (_PyUnion_Check(type)) {
+            else if (_PyUnion_Check(type)) {
                 // optional field
                 // do nothing, we'll have set a None default on the class
             }
@@ -1026,8 +1030,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
                     "This will become an error in Python 3.15.",
                     Py_TYPE(self)->tp_name, name
                 ) < 0) {
-                    res = -1;
-                    goto cleanup;
+                    goto set_remaining_cleanup;
                 }
             }
         }
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 1953142..4956d04 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -5178,14 +5178,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
             goto cleanup;
         }
         if (field_types == NULL) {
-            if (PyErr_WarnFormat(
-                PyExc_DeprecationWarning, 1,
-                "%.400s provides _fields but not _field_types. "
-                "This will become an error in Python 3.15.",
-                Py_TYPE(self)->tp_name
-            ) < 0) {
-                res = -1;
-            }
+            // Probably a user-defined subclass of AST that lacks _field_types.
+            // This will continue to work as it did before 3.13; i.e., attributes
+            // that are not passed in simply do not exist on the instance.
             goto cleanup;
         }
         remaining_list = PySequence_List(remaining_fields);
@@ -5196,12 +5191,21 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
             PyObject *name = PyList_GET_ITEM(remaining_list, i);
             PyObject *type = PyDict_GetItemWithError(field_types, name);
             if (!type) {
-                if (!PyErr_Occurred()) {
-                    PyErr_SetObject(PyExc_KeyError, name);
+                if (PyErr_Occurred()) {
+                    goto set_remaining_cleanup;
+                }
+                else {
+                    if (PyErr_WarnFormat(
+                        PyExc_DeprecationWarning, 1,
+                        "Field '%U' is missing from %.400s._field_types. "
+                        "This will become an error in Python 3.15.",
+                        name, Py_TYPE(self)->tp_name
+                    ) < 0) {
+                        goto set_remaining_cleanup;
+                    }
                 }
-                goto set_remaining_cleanup;
             }
-            if (_PyUnion_Check(type)) {
+            else if (_PyUnion_Check(type)) {
                 // optional field
                 // do nothing, we'll have set a None default on the class
             }
@@ -5225,8 +5229,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw)
                     "This will become an error in Python 3.15.",
                     Py_TYPE(self)->tp_name, name
                 ) < 0) {
-                    res = -1;
-                    goto cleanup;
+                    goto set_remaining_cleanup;
                 }
             }
         }