bpo-36143: Regenerate Lib/keyword.py from the Grammar and Tokens file using pgen (GH-12456)

Now that the parser generator is written in Python (Parser/pgen) we can make use of it to regenerate the Lib/keyword file that contains the language keywords instead of parsing the autogenerated grammar files. This also allows checking in the CI that the autogenerated files are up to date.

1 files changed, 43 insertions, 86 deletions

diff --git a/Lib/keyword.py b/Lib/keyword.py
index 150c2bc..ddcbb25 100755..100644
--- a/Lib/keyword.py
+++ b/Lib/keyword.py
@@ -1,98 +1,55 @@
-#! /usr/bin/env python3
-
-"""Keywords (from "graminit.c")
+"""Keywords (from "Grammar/Grammar")
 
 This file is automatically generated; please don't muck it up!
 
 To update the symbols in this file, 'cd' to the top directory of
-the python source tree after building the interpreter and run:
+the python source tree and run:
+
+    python3 -m Parser.pgen.keywordgen Grammar/Grammar \
+                                      Grammar/Tokens \
+                                      Lib/keyword.py
 
-    ./python Lib/keyword.py
+Alternatively, you can run 'make regen-keyword'.
 """
 
 __all__ = ["iskeyword", "kwlist"]
 
 kwlist = [
-#--start keywords--
-        'False',
-        'None',
-        'True',
-        'and',
-        'as',
-        'assert',
-        'break',
-        'class',
-        'continue',
-        'def',
-        'del',
-        'elif',
-        'else',
-        'except',
-        'finally',
-        'for',
-        'from',
-        'global',
-        'if',
-        'import',
-        'in',
-        'is',
-        'lambda',
-        'nonlocal',
-        'not',
-        'or',
-        'pass',
-        'raise',
-        'return',
-        'try',
-        'while',
-        'with',
-        'yield',
-#--end keywords--
-        ]
-
-kwlist.append('async')
-kwlist.append('await')
-kwlist.sort()
+    'False',
+    'None',
+    'True',
+    'and',
+    'as',
+    'assert',
+    'async',
+    'await',
+    'break',
+    'class',
+    'continue',
+    'def',
+    'del',
+    'elif',
+    'else',
+    'except',
+    'finally',
+    'for',
+    'from',
+    'global',
+    'if',
+    'import',
+    'in',
+    'is',
+    'lambda',
+    'nonlocal',
+    'not',
+    'or',
+    'pass',
+    'raise',
+    'return',
+    'try',
+    'while',
+    'with',
+    'yield'
+]
 
 iskeyword = frozenset(kwlist).__contains__
-
-def main():
-    import sys, re
-
-    args = sys.argv[1:]
-    iptfile = args and args[0] or "Python/graminit.c"
-    if len(args) > 1: optfile = args[1]
-    else: optfile = "Lib/keyword.py"
-
-    # load the output skeleton from the target, taking care to preserve its
-    # newline convention.
-    with open(optfile, newline='') as fp:
-        format = fp.readlines()
-    nl = format[0][len(format[0].strip()):] if format else '\n'
-
-    # scan the source file for keywords
-    with open(iptfile) as fp:
-        strprog = re.compile('"([^"]+)"')
-        lines = []
-        for line in fp:
-            if '{1, "' in line:
-                match = strprog.search(line)
-                if match:
-                    lines.append("        '" + match.group(1) + "'," + nl)
-    lines.sort()
-
-    # insert the lines of keywords into the skeleton
-    try:
-        start = format.index("#--start keywords--" + nl) + 1
-        end = format.index("#--end keywords--" + nl)
-        format[start:end] = lines
-    except ValueError:
-        sys.stderr.write("target does not contain format markers\n")
-        sys.exit(1)
-
-    # write the output file
-    with open(optfile, 'w', newline='') as fp:
-        fp.writelines(format)
-
-if __name__ == "__main__":
-    main()