From e55a0e971b4bf9e6473bf0ca6bebdff76c075ef6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 3 Sep 2020 15:29:55 +0100 Subject: Fix 'gather' rules in the python parser generator (GH-22021) Currently, empty sequences in gather rules make the conditional for gather rules fail as empty sequences evaluate as "False". We need to explicitly check for "None" (the failure condition) to avoid false negatives. --- Lib/test/test_peg_generator/test_pegen.py | 16 +++++++++++++++- Tools/peg_generator/pegen/python_generator.py | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 5b4e964..bcfee3f 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -74,7 +74,7 @@ class TestPegen(unittest.TestCase): "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" ) - def test_repeat_with_separator_rules(self) -> None: + def test_gather(self) -> None: grammar = """ start: ','.thing+ NEWLINE thing: NUMBER @@ -85,6 +85,20 @@ class TestPegen(unittest.TestCase): "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" )) self.assertEqual(str(rules["thing"]), "thing: NUMBER") + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + assert node == [ + [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + ] + node = parse_string("1, 2\n", parser_class) + assert node == [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")], + [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"), + ] def test_expr_grammar(self) -> None: grammar = """ diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 45a7597..b786de7 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -217,6 +217,9 @@ class PythonParserGenerator(ParserGenerator, GrammarVisitor): else: self.print("and") self.visit(item) + if is_gather: + self.print("is not None") + self.print("):") with self.indent(): action = node.action -- cgit v0.12