bpo-43244: Fix test_peg_generator for PyAST_Validate() (GH-24912)

test_peg_generator now defines _Py_TEST_PEGEN macro when building C
code to not call PyAST_Validate() in Parser/pegen.c. Moreover, it
defines Py_BUILD_CORE_MODULE macro to get access to the internal
C API.

Remove "global_ast_state" from Python-ast.c when it's built by
test_peg_generator: always get the AST state from the current interpreter.

4 files changed, 14 insertions, 282 deletions

diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index e96f1f3..aefea12 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1373,17 +1373,13 @@ def generate_ast_fini(module_state, f):
     f.write(textwrap.dedent("""
             void _PyAST_Fini(PyInterpreterState *interp)
             {
-            #ifdef Py_BUILD_CORE
                 struct ast_state *state = &interp->ast;
-            #else
-                struct ast_state *state = &global_ast_state;
-            #endif
 
     """))
     for s in module_state:
         f.write("    Py_CLEAR(state->" + s + ');\n')
     f.write(textwrap.dedent("""
-            #if defined(Py_BUILD_CORE) && !defined(NDEBUG)
+            #if !defined(NDEBUG)
                 state->initialized = -1;
             #else
                 state->initialized = 0;
@@ -1428,24 +1424,15 @@ def generate_module_def(mod, f, internal_h):
     generate_ast_state(module_state, internal_h)
 
     print(textwrap.dedent(f"""
-        #ifdef Py_BUILD_CORE
-        #  include "pycore_ast_state.h"     // struct ast_state
-        #  include "pycore_interp.h"        // _PyInterpreterState.ast
-        #  include "pycore_pystate.h"       // _PyInterpreterState_GET()
-        #else
-    """).strip(), file=f)
-
-    generate_ast_state(module_state, f)
-
-    print(textwrap.dedent(f"""
-        #endif   // Py_BUILD_CORE
+        #include "pycore_ast_state.h"       // struct ast_state
+        #include "pycore_interp.h"          // _PyInterpreterState.ast
+        #include "pycore_pystate.h"         // _PyInterpreterState_GET()
     """).rstrip(), file=f)
 
     f.write("""
 // Forward declaration
 static int init_types(struct ast_state *state);
 
-#ifdef Py_BUILD_CORE
 static struct ast_state*
 get_ast_state(void)
 {
@@ -1456,19 +1443,6 @@ get_ast_state(void)
     }
     return state;
 }
-#else
-static struct ast_state global_ast_state;
-
-static struct ast_state*
-get_ast_state(void)
-{
-    struct ast_state *state = &global_ast_state;
-    if (!init_types(state)) {
-        return NULL;
-    }
-    return state;
-}
-#endif   // Py_BUILD_CORE
 """)
 
     # f-string for {mod.name}
diff --git a/Parser/pegen.c b/Parser/pegen.c
index 3011993..84bdf8d 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -1265,7 +1265,8 @@ _PyPegen_run_parser(Parser *p)
         return RAISE_SYNTAX_ERROR("multiple statements found while compiling a single statement");
     }
 
-#if defined(Py_DEBUG) && defined(Py_BUILD_CORE)
+    // test_peg_generator defines _Py_TEST_PEGEN to not call PyAST_Validate()
+#if defined(Py_DEBUG) && !defined(_Py_TEST_PEGEN)
     if (p->start_rule == Py_single_input ||
         p->start_rule == Py_file_input ||
         p->start_rule == Py_eval_input)
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index 59acce7..4524877 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -5,243 +5,14 @@
 
 #include "Python.h"
 
-#ifdef Py_BUILD_CORE
-#  include "pycore_ast_state.h"     // struct ast_state
-#  include "pycore_interp.h"        // _PyInterpreterState.ast
-#  include "pycore_pystate.h"       // _PyInterpreterState_GET()
-#else
-struct ast_state {
-    int initialized;
-    PyObject *AST_type;
-    PyObject *Add_singleton;
-    PyObject *Add_type;
-    PyObject *And_singleton;
-    PyObject *And_type;
-    PyObject *AnnAssign_type;
-    PyObject *Assert_type;
-    PyObject *Assign_type;
-    PyObject *AsyncFor_type;
-    PyObject *AsyncFunctionDef_type;
-    PyObject *AsyncWith_type;
-    PyObject *Attribute_type;
-    PyObject *AugAssign_type;
-    PyObject *Await_type;
-    PyObject *BinOp_type;
-    PyObject *BitAnd_singleton;
-    PyObject *BitAnd_type;
-    PyObject *BitOr_singleton;
-    PyObject *BitOr_type;
-    PyObject *BitXor_singleton;
-    PyObject *BitXor_type;
-    PyObject *BoolOp_type;
-    PyObject *Break_type;
-    PyObject *Call_type;
-    PyObject *ClassDef_type;
-    PyObject *Compare_type;
-    PyObject *Constant_type;
-    PyObject *Continue_type;
-    PyObject *Del_singleton;
-    PyObject *Del_type;
-    PyObject *Delete_type;
-    PyObject *DictComp_type;
-    PyObject *Dict_type;
-    PyObject *Div_singleton;
-    PyObject *Div_type;
-    PyObject *Eq_singleton;
-    PyObject *Eq_type;
-    PyObject *ExceptHandler_type;
-    PyObject *Expr_type;
-    PyObject *Expression_type;
-    PyObject *FloorDiv_singleton;
-    PyObject *FloorDiv_type;
-    PyObject *For_type;
-    PyObject *FormattedValue_type;
-    PyObject *FunctionDef_type;
-    PyObject *FunctionType_type;
-    PyObject *GeneratorExp_type;
-    PyObject *Global_type;
-    PyObject *GtE_singleton;
-    PyObject *GtE_type;
-    PyObject *Gt_singleton;
-    PyObject *Gt_type;
-    PyObject *IfExp_type;
-    PyObject *If_type;
-    PyObject *ImportFrom_type;
-    PyObject *Import_type;
-    PyObject *In_singleton;
-    PyObject *In_type;
-    PyObject *Interactive_type;
-    PyObject *Invert_singleton;
-    PyObject *Invert_type;
-    PyObject *IsNot_singleton;
-    PyObject *IsNot_type;
-    PyObject *Is_singleton;
-    PyObject *Is_type;
-    PyObject *JoinedStr_type;
-    PyObject *LShift_singleton;
-    PyObject *LShift_type;
-    PyObject *Lambda_type;
-    PyObject *ListComp_type;
-    PyObject *List_type;
-    PyObject *Load_singleton;
-    PyObject *Load_type;
-    PyObject *LtE_singleton;
-    PyObject *LtE_type;
-    PyObject *Lt_singleton;
-    PyObject *Lt_type;
-    PyObject *MatMult_singleton;
-    PyObject *MatMult_type;
-    PyObject *MatchAs_type;
-    PyObject *MatchOr_type;
-    PyObject *Match_type;
-    PyObject *Mod_singleton;
-    PyObject *Mod_type;
-    PyObject *Module_type;
-    PyObject *Mult_singleton;
-    PyObject *Mult_type;
-    PyObject *Name_type;
-    PyObject *NamedExpr_type;
-    PyObject *Nonlocal_type;
-    PyObject *NotEq_singleton;
-    PyObject *NotEq_type;
-    PyObject *NotIn_singleton;
-    PyObject *NotIn_type;
-    PyObject *Not_singleton;
-    PyObject *Not_type;
-    PyObject *Or_singleton;
-    PyObject *Or_type;
-    PyObject *Pass_type;
-    PyObject *Pow_singleton;
-    PyObject *Pow_type;
-    PyObject *RShift_singleton;
-    PyObject *RShift_type;
-    PyObject *Raise_type;
-    PyObject *Return_type;
-    PyObject *SetComp_type;
-    PyObject *Set_type;
-    PyObject *Slice_type;
-    PyObject *Starred_type;
-    PyObject *Store_singleton;
-    PyObject *Store_type;
-    PyObject *Sub_singleton;
-    PyObject *Sub_type;
-    PyObject *Subscript_type;
-    PyObject *Try_type;
-    PyObject *Tuple_type;
-    PyObject *TypeIgnore_type;
-    PyObject *UAdd_singleton;
-    PyObject *UAdd_type;
-    PyObject *USub_singleton;
-    PyObject *USub_type;
-    PyObject *UnaryOp_type;
-    PyObject *While_type;
-    PyObject *With_type;
-    PyObject *YieldFrom_type;
-    PyObject *Yield_type;
-    PyObject *__dict__;
-    PyObject *__doc__;
-    PyObject *__match_args__;
-    PyObject *__module__;
-    PyObject *_attributes;
-    PyObject *_fields;
-    PyObject *alias_type;
-    PyObject *annotation;
-    PyObject *arg;
-    PyObject *arg_type;
-    PyObject *args;
-    PyObject *argtypes;
-    PyObject *arguments_type;
-    PyObject *asname;
-    PyObject *ast;
-    PyObject *attr;
-    PyObject *bases;
-    PyObject *body;
-    PyObject *boolop_type;
-    PyObject *cases;
-    PyObject *cause;
-    PyObject *cmpop_type;
-    PyObject *col_offset;
-    PyObject *comparators;
-    PyObject *comprehension_type;
-    PyObject *context_expr;
-    PyObject *conversion;
-    PyObject *ctx;
-    PyObject *decorator_list;
-    PyObject *defaults;
-    PyObject *elt;
-    PyObject *elts;
-    PyObject *end_col_offset;
-    PyObject *end_lineno;
-    PyObject *exc;
-    PyObject *excepthandler_type;
-    PyObject *expr_context_type;
-    PyObject *expr_type;
-    PyObject *finalbody;
-    PyObject *format_spec;
-    PyObject *func;
-    PyObject *generators;
-    PyObject *guard;
-    PyObject *handlers;
-    PyObject *id;
-    PyObject *ifs;
-    PyObject *is_async;
-    PyObject *items;
-    PyObject *iter;
-    PyObject *key;
-    PyObject *keys;
-    PyObject *keyword_type;
-    PyObject *keywords;
-    PyObject *kind;
-    PyObject *kw_defaults;
-    PyObject *kwarg;
-    PyObject *kwonlyargs;
-    PyObject *left;
-    PyObject *level;
-    PyObject *lineno;
-    PyObject *lower;
-    PyObject *match_case_type;
-    PyObject *mod_type;
-    PyObject *module;
-    PyObject *msg;
-    PyObject *name;
-    PyObject *names;
-    PyObject *op;
-    PyObject *operand;
-    PyObject *operator_type;
-    PyObject *ops;
-    PyObject *optional_vars;
-    PyObject *orelse;
-    PyObject *pattern;
-    PyObject *patterns;
-    PyObject *posonlyargs;
-    PyObject *returns;
-    PyObject *right;
-    PyObject *simple;
-    PyObject *slice;
-    PyObject *step;
-    PyObject *stmt_type;
-    PyObject *subject;
-    PyObject *tag;
-    PyObject *target;
-    PyObject *targets;
-    PyObject *test;
-    PyObject *type;
-    PyObject *type_comment;
-    PyObject *type_ignore_type;
-    PyObject *type_ignores;
-    PyObject *unaryop_type;
-    PyObject *upper;
-    PyObject *value;
-    PyObject *values;
-    PyObject *vararg;
-    PyObject *withitem_type;
-};
-#endif   // Py_BUILD_CORE
+
+#include "pycore_ast_state.h"       // struct ast_state
+#include "pycore_interp.h"          // _PyInterpreterState.ast
+#include "pycore_pystate.h"         // _PyInterpreterState_GET()
 
 // Forward declaration
 static int init_types(struct ast_state *state);
 
-#ifdef Py_BUILD_CORE
 static struct ast_state*
 get_ast_state(void)
 {
@@ -252,19 +23,6 @@ get_ast_state(void)
     }
     return state;
 }
-#else
-static struct ast_state global_ast_state;
-
-static struct ast_state*
-get_ast_state(void)
-{
-    struct ast_state *state = &global_ast_state;
-    if (!init_types(state)) {
-        return NULL;
-    }
-    return state;
-}
-#endif   // Py_BUILD_CORE
 
 // Include Python-ast.h after pycore_interp.h to avoid conflicts
 // with the Yield macro redefined by <winbase.h>
@@ -273,11 +31,7 @@ get_ast_state(void)
 
 void _PyAST_Fini(PyInterpreterState *interp)
 {
-#ifdef Py_BUILD_CORE
     struct ast_state *state = &interp->ast;
-#else
-    struct ast_state *state = &global_ast_state;
-#endif
 
     Py_CLEAR(state->AST_type);
     Py_CLEAR(state->Add_singleton);
@@ -503,7 +257,7 @@ void _PyAST_Fini(PyInterpreterState *interp)
     Py_CLEAR(state->vararg);
     Py_CLEAR(state->withitem_type);
 
-#if defined(Py_BUILD_CORE) && !defined(NDEBUG)
+#if !defined(NDEBUG)
     state->initialized = -1;
 #else
     state->initialized = 0;
diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py
index 9edde37..b80fc85 100644
--- a/Tools/peg_generator/pegen/build.py
+++ b/Tools/peg_generator/pegen/build.py
@@ -56,6 +56,9 @@ def compile_c_extension(
     source_file_path = pathlib.Path(generated_source_path)
     extension_name = source_file_path.stem
     extra_compile_args = get_extra_flags("CFLAGS", "PY_CFLAGS_NODIST")
+    extra_compile_args.append("-DPy_BUILD_CORE_MODULE")
+    # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c
+    extra_compile_args.append('-D_Py_TEST_PEGEN')
     extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST")
     if keep_asserts:
         extra_compile_args.append("-UNDEBUG")