From f5f7b54e5ead8aba2fe9a1a494856d494b14a84a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 30 Aug 2020 20:57:14 +0100 Subject: [PATCH] Fix 'gather' rules in the python parser generator Currently, empty sequences in gather rules make the conditional for gather rules fail as empty sequences evaluate as "False". We need to explicitly check for "None" (the failure condition) to avoid false negatives. --- Lib/test/test_peg_generator/test_pegen.py | 16 +++++++++++++++- Tools/peg_generator/pegen/python_generator.py | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 5b4e964d698ade..bcfee3f2c5f8c3 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -74,7 +74,7 @@ def test_typed_rules(self) -> None: "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" ) - def test_repeat_with_separator_rules(self) -> None: + def test_gather(self) -> None: grammar = """ start: ','.thing+ NEWLINE thing: NUMBER @@ -85,6 +85,20 @@ def test_repeat_with_separator_rules(self) -> None: "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" )) self.assertEqual(str(rules["thing"]), "thing: NUMBER") + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + assert node == [ + [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + ] + node = parse_string("1, 2\n", parser_class) + assert node == [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")], + [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"), + ] def test_expr_grammar(self) -> None: grammar = """ diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 45a75975dbf5e0..b786de7fee5b43 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -217,6 +217,9 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: else: self.print("and") self.visit(item) + if is_gather: + self.print("is not None") + self.print("):") with self.indent(): action = node.action