bpo-40334: Refactor peg_generator to receive a Tokens file when building c code (GH-19745)

1 files changed, 12 insertions, 5 deletions

diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py
index 06a38fc..6511a2d 100755
--- a/Tools/peg_generator/scripts/test_parse_directory.py
+++ b/Tools/peg_generator/scripts/test_parse_directory.py
@@ -13,7 +13,7 @@ from pathlib import PurePath
 from typing import List, Optional, Any
 
 sys.path.insert(0, os.getcwd())
-from pegen.build import build_parser_and_generator
+from pegen.build import build_c_parser_and_generator
 from pegen.testutil import print_memstats
 from scripts import show_parse
 
@@ -26,7 +26,8 @@ argparser = argparse.ArgumentParser(
     description="Helper program to test directories or files for pegen",
 )
 argparser.add_argument("-d", "--directory", help="Directory path containing files to test")
-argparser.add_argument("-g", "--grammar-file", help="Grammar file path")
+argparser.add_argument("--grammar-file", help="Grammar file path")
+argparser.add_argument("--tokens-file", help="Tokens file path")
 argparser.add_argument(
     "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude"
 )
@@ -114,6 +115,7 @@ def compare_trees(
 def parse_directory(
     directory: str,
     grammar_file: str,
+    tokens_file: str,
     verbose: bool,
     excluded_files: List[str],
     skip_actions: bool,
@@ -131,15 +133,16 @@ def parse_directory(
         print("You must specify a directory of files to test.", file=sys.stderr)
         return 1
 
-    if grammar_file:
+    if grammar_file and tokens_file:
         if not os.path.exists(grammar_file):
             print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
             return 1
 
         try:
             if not extension and parser == "pegen":
-                build_parser_and_generator(
+                build_c_parser_and_generator(
                     grammar_file,
+                    tokens_file,
                     "peg_extension/parse.c",
                     compile_extension=True,
                     skip_actions=skip_actions,
@@ -154,7 +157,9 @@ def parse_directory(
             return 1
 
     else:
-        print("A grammar file was not provided - attempting to use existing file...\n")
+        print(
+            "A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n"
+        )
 
     if parser == "pegen":
         try:
@@ -264,6 +269,7 @@ def main() -> None:
     args = argparser.parse_args()
     directory = args.directory
     grammar_file = args.grammar_file
+    tokens_file = args.tokens_file
     verbose = args.verbose
     excluded_files = args.exclude
     skip_actions = args.skip_actions
@@ -273,6 +279,7 @@ def main() -> None:
         parse_directory(
             directory,
             grammar_file,
+            tokens_file,
             verbose,
             excluded_files,
             skip_actions,