bpo-41796: Call _PyAST_Fini() earlier to fix a leak (GH-23131)

Call _PyAST_Fini() on all interpreters, not only on the main
interpreter. Also, call it ealier to fix a reference leak.

Python types contain a reference to themselves in in their
PyTypeObject.tp_mro member. _PyAST_Fini() must called before the last
GC collection to destroy AST types.

_PyInterpreterState_Clear() now calls _PyAST_Fini(). It now also
calls _PyWarnings_Fini() on subinterpeters, not only on the main
interpreter.

Add an assertion in AST init_types() to ensure that the _ast module
is no longer used after _PyAST_Fini() has been called.

3 files changed, 33 insertions, 19 deletions

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index f04addb..a456b51 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -261,10 +261,10 @@ get_ast_state(void)
 #include "Python-ast.h"
 #include "structmember.h"
 
-void _PyAST_Fini(PyThreadState *tstate)
+void _PyAST_Fini(PyInterpreterState *interp)
 {
 #ifdef Py_BUILD_CORE
-    struct ast_state *state = &tstate->interp->ast;
+    struct ast_state *state = &interp->ast;
 #else
     struct ast_state *state = &global_ast_state;
 #endif
@@ -483,7 +483,11 @@ void _PyAST_Fini(PyThreadState *tstate)
     Py_CLEAR(state->vararg);
     Py_CLEAR(state->withitem_type);
 
+#if defined(Py_BUILD_CORE) && !defined(NDEBUG)
+    state->initialized = -1;
+#else
     state->initialized = 0;
+#endif
 }
 
 static int init_identifiers(struct ast_state *state)
@@ -1227,13 +1231,27 @@ static int add_ast_fields(struct ast_state *state)
 }
 
 
-static int init_types(struct ast_state *state)
+
+static int
+init_types(struct ast_state *state)
 {
-    if (state->initialized) return 1;
-    if (init_identifiers(state) < 0) return 0;
+    // init_types() must not be called after _PyAST_Fini()
+    // has been called
+    assert(state->initialized >= 0);
+
+    if (state->initialized) {
+        return 1;
+    }
+    if (init_identifiers(state) < 0) {
+        return 0;
+    }
     state->AST_type = PyType_FromSpec(&AST_type_spec);
-    if (!state->AST_type) return 0;
-    if (add_ast_fields(state) < 0) return 0;
+    if (!state->AST_type) {
+        return 0;
+    }
+    if (add_ast_fields(state) < 0) {
+        return 0;
+    }
     state->mod_type = make_type(state, "mod", state->AST_type, NULL, 0,
         "mod = Module(stmt* body, type_ignore* type_ignores)\n"
         "    | Interactive(stmt* body)\n"
@@ -1902,6 +1920,7 @@ static int init_types(struct ast_state *state)
                                        TypeIgnore_fields, 2,
         "TypeIgnore(int lineno, string tag)");
     if (!state->TypeIgnore_type) return 0;
+
     state->initialized = 1;
     return 1;
 }
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index ff58c1b..cad0fa7 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1545,12 +1545,6 @@ flush_std_files(void)
 static void
 finalize_interp_types(PyThreadState *tstate)
 {
-    // The _ast module state is shared by all interpreters.
-    // The state must only be cleared by the main interpreter.
-    if (_Py_IsMainInterpreter(tstate)) {
-        _PyAST_Fini(tstate);
-    }
-
     _PyExc_Fini(tstate);
     _PyFrame_Fini(tstate);
     _PyAsyncGen_Fini(tstate);
@@ -1591,8 +1585,6 @@ finalize_interp_clear(PyThreadState *tstate)
         _Py_ClearFileSystemEncoding();
     }
 
-    _PyWarnings_Fini(tstate->interp);
-
     finalize_interp_types(tstate);
 }
 
diff --git a/Python/pystate.c b/Python/pystate.c
index e37cbd5..c9882a7 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -300,13 +300,16 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
     Py_CLEAR(interp->after_forkers_parent);
     Py_CLEAR(interp->after_forkers_child);
 #endif
-    if (_PyRuntimeState_GetFinalizing(runtime) == NULL) {
-        _PyWarnings_Fini(interp);
-    }
+
+    _PyAST_Fini(interp);
+    _PyWarnings_Fini(interp);
+
+    // All Python types must be destroyed before the last GC collection. Python
+    // types create a reference cycle to themselves in their in their
+    // PyTypeObject.tp_mro member (the tuple contains the type).
 
     /* Last garbage collection on this interpreter */
     _PyGC_CollectNoFail(tstate);
-
     _PyGC_Fini(tstate);
 
     /* We don't clear sysdict and builtins until the end of this function.