diff options
Diffstat (limited to 'Tools/peg_generator/pegen/testutil.py')
| -rw-r--r-- | Tools/peg_generator/pegen/testutil.py | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py index 5a91862..1f79d8f 100644 --- a/Tools/peg_generator/pegen/testutil.py +++ b/Tools/peg_generator/pegen/testutil.py @@ -5,6 +5,7 @@ import pathlib import sys import textwrap import tokenize +import token from typing import Any, cast, Dict, IO, Type, Final @@ -16,6 +17,11 @@ from pegen.parser import Parser from pegen.python_generator import PythonParserGenerator from pegen.tokenizer import Tokenizer +EXACT_TOKENS = token.EXACT_TOKEN_TYPES # type: ignore +NON_EXACT_TOKENS = { + name for index, name in token.tok_name.items() if index not in EXACT_TOKENS.values() +} + def generate_parser(grammar: Grammar) -> Type[Parser]: # Generate a parser. @@ -70,7 +76,7 @@ def import_file(full_name: str, path: str) -> Any: def generate_c_parser_source(grammar: Grammar) -> str: out = io.StringIO() - genr = CParserGenerator(grammar, out) + genr = CParserGenerator(grammar, EXACT_TOKENS, NON_EXACT_TOKENS, out) genr.generate("<string>") return out.getvalue() @@ -90,7 +96,7 @@ def generate_parser_c_extension( assert not os.listdir(path) source = path / "parse.c" with open(source, "w") as file: - genr = CParserGenerator(grammar, file, debug=debug) + genr = CParserGenerator(grammar, EXACT_TOKENS, NON_EXACT_TOKENS, file, debug=debug) genr.generate("parse.c") compile_c_extension(str(source), build_dir=str(path)) |