diff options
| -rw-r--r-- | Lib/test/test_peg_generator/test_pegen.py | 16 | ||||
| -rw-r--r-- | Tools/peg_generator/pegen/python_generator.py | 3 |
2 files changed, 18 insertions, 1 deletions
diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 5b4e964..bcfee3f 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -74,7 +74,7 @@ class TestPegen(unittest.TestCase): "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" ) - def test_repeat_with_separator_rules(self) -> None: + def test_gather(self) -> None: grammar = """ start: ','.thing+ NEWLINE thing: NUMBER @@ -85,6 +85,20 @@ class TestPegen(unittest.TestCase): "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" )) self.assertEqual(str(rules["thing"]), "thing: NUMBER") + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + assert node == [ + [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + ] + node = parse_string("1, 2\n", parser_class) + assert node == [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")], + [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"), + ] def test_expr_grammar(self) -> None: grammar = """ diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 45a7597..b786de7 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -217,6 +217,9 @@ class PythonParserGenerator(ParserGenerator, GrammarVisitor): else: self.print("and") self.visit(item) + if is_gather: + self.print("is not None") + self.print("):") with self.indent(): action = node.action |