diff options
Diffstat (limited to 'Parser/pgen/token.py')
| -rw-r--r-- | Parser/pgen/token.py | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/Parser/pgen/token.py b/Parser/pgen/token.py index f9d45c4..008e241 100644 --- a/Parser/pgen/token.py +++ b/Parser/pgen/token.py @@ -1,5 +1,6 @@ import itertools + def generate_tokens(tokens): numbers = itertools.count(0) for line in tokens: @@ -16,6 +17,7 @@ def generate_tokens(tokens): yield ('N_TOKENS', next(numbers)) yield ('NT_OFFSET', 256) + def generate_opmap(tokens): for line in tokens: line = line.strip() |