bpo-40528: Improve AST generation script to do builds simultaneously (GH-19968)

- Switch from getopt to argparse.
- Removed the limitation of not being able to produce both C and H simultaneously.

This will make it run faster since it parses the asdl definition once and uses the generated tree to generate both the header and the C source.

1 files changed, 73 insertions, 85 deletions

diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 59bf03e..6d57275 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1,12 +1,17 @@
 #! /usr/bin/env python
 """Generate C code from an ASDL description."""
 
-import os, sys
+import os
+import sys
+
+from argparse import ArgumentParser
+from pathlib import Path
 
 import asdl
 
 TABSIZE = 4
 MAX_COL = 80
+AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n"
 
 def get_c_type(name):
     """Return a string for the C name of the type.
@@ -1369,94 +1374,77 @@ static struct PyModuleDef _astmodule = {
     f.write('    return 1;\n')
     f.write('};\n\n')
 
-
-common_msg = "/* File automatically generated by %s. */\n\n"
-
-def main(srcfile, dump_module=False):
-    argv0 = sys.argv[0]
-    components = argv0.split(os.sep)
-    # Always join with '/' so different OS does not keep changing the file
-    argv0 = '/'.join(components[-2:])
-    auto_gen_msg = common_msg % argv0
-    mod = asdl.parse(srcfile)
+def write_header(f, mod):
+    f.write('#ifndef Py_PYTHON_AST_H\n')
+    f.write('#define Py_PYTHON_AST_H\n')
+    f.write('#ifdef __cplusplus\n')
+    f.write('extern "C" {\n')
+    f.write('#endif\n')
+    f.write('\n')
+    f.write('#ifndef Py_LIMITED_API\n')
+    f.write('#include "asdl.h"\n')
+    f.write('\n')
+    f.write('#undef Yield   /* undefine macro conflicting with <winbase.h> */\n')
+    f.write('\n')
+    c = ChainOfVisitors(TypeDefVisitor(f),
+                        StructVisitor(f))
+    c.visit(mod)
+    f.write("// Note: these macros affect function definitions, not only call sites.\n")
+    PrototypeVisitor(f).visit(mod)
+    f.write("\n")
+    f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
+    f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
+    f.write("int PyAST_Check(PyObject* obj);\n")
+    f.write("#endif /* !Py_LIMITED_API */\n")
+    f.write('\n')
+    f.write('#ifdef __cplusplus\n')
+    f.write('}\n')
+    f.write('#endif\n')
+    f.write('#endif /* !Py_PYTHON_AST_H */\n')
+
+def write_source(f, mod):
+    f.write('#include <stddef.h>\n')
+    f.write('\n')
+    f.write('#include "Python.h"\n')
+    f.write('#include "%s-ast.h"\n' % mod.name)
+    f.write('#include "structmember.h"         // PyMemberDef\n')
+    f.write('\n')
+
+    generate_module_def(f, mod)
+
+    v = ChainOfVisitors(
+        PyTypesDeclareVisitor(f),
+        PyTypesVisitor(f),
+        Obj2ModPrototypeVisitor(f),
+        FunctionVisitor(f),
+        ObjVisitor(f),
+        Obj2ModVisitor(f),
+        ASTModuleVisitor(f),
+        PartingShots(f),
+    )
+    v.visit(mod)
+
+def main(input_file, c_file, h_file, dump_module=False):
+    auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
+    mod = asdl.parse(input_file)
     if dump_module:
         print('Parsed Module:')
         print(mod)
     if not asdl.check(mod):
         sys.exit(1)
-    if H_FILE:
-        with open(H_FILE, "w") as f:
-            f.write(auto_gen_msg)
-            f.write('#ifndef Py_PYTHON_AST_H\n')
-            f.write('#define Py_PYTHON_AST_H\n')
-            f.write('#ifdef __cplusplus\n')
-            f.write('extern "C" {\n')
-            f.write('#endif\n')
-            f.write('\n')
-            f.write('#ifndef Py_LIMITED_API\n')
-            f.write('#include "asdl.h"\n')
-            f.write('\n')
-            f.write('#undef Yield   /* undefine macro conflicting with <winbase.h> */\n')
-            f.write('\n')
-            c = ChainOfVisitors(TypeDefVisitor(f),
-                                StructVisitor(f))
-
-            c.visit(mod)
-            f.write("// Note: these macros affect function definitions, not only call sites.\n")
-            PrototypeVisitor(f).visit(mod)
-            f.write("\n")
-            f.write("PyObject* PyAST_mod2obj(mod_ty t);\n")
-            f.write("mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode);\n")
-            f.write("int PyAST_Check(PyObject* obj);\n")
-            f.write("#endif /* !Py_LIMITED_API */\n")
-            f.write('\n')
-            f.write('#ifdef __cplusplus\n')
-            f.write('}\n')
-            f.write('#endif\n')
-            f.write('#endif /* !Py_PYTHON_AST_H */\n')
-
-    if C_FILE:
-        with open(C_FILE, "w") as f:
-            f.write(auto_gen_msg)
-            f.write('#include <stddef.h>\n')
-            f.write('\n')
-            f.write('#include "Python.h"\n')
-            f.write('#include "%s-ast.h"\n' % mod.name)
-            f.write('#include "structmember.h"         // PyMemberDef\n')
-            f.write('\n')
-
-            generate_module_def(f, mod)
-
-            v = ChainOfVisitors(
-                PyTypesDeclareVisitor(f),
-                PyTypesVisitor(f),
-                Obj2ModPrototypeVisitor(f),
-                FunctionVisitor(f),
-                ObjVisitor(f),
-                Obj2ModVisitor(f),
-                ASTModuleVisitor(f),
-                PartingShots(f),
-                )
-            v.visit(mod)
+    for file, writer in (c_file, write_source), (h_file, write_header):
+        if file is not None:
+            with file.open("w") as f:
+                f.write(auto_gen_msg)
+                writer(f, mod)
+            print(file, "regenerated.")
 
 if __name__ == "__main__":
-    import getopt
-
-    H_FILE = ''
-    C_FILE = ''
-    dump_module = False
-    opts, args = getopt.getopt(sys.argv[1:], "dh:c:")
-    for o, v in opts:
-        if o == '-h':
-            H_FILE = v
-        elif o == '-c':
-            C_FILE = v
-        elif o == '-d':
-            dump_module = True
-    if H_FILE and C_FILE:
-        print('Must specify exactly one output file')
-        sys.exit(1)
-    elif len(args) != 1:
-        print('Must specify single input file')
-        sys.exit(1)
-    main(args[0], dump_module)
+    parser = ArgumentParser()
+    parser.add_argument("input_file", type=Path)
+    parser.add_argument("-C", "--c-file", type=Path, default=None)
+    parser.add_argument("-H", "--h-file", type=Path, default=None)
+    parser.add_argument("-d", "--dump-module", action="store_true")
+
+    options = parser.parse_args()
+    main(**vars(options))