From a0eb9e0d4677b65191dfaf4eb99c2923d4882d65 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 3 Sep 2020 15:29:55 +0100 Subject: [PATCH] [3.9] Fix 'gather' rules in the python parser generator (GH-22021) Currently, empty sequences in gather rules make the conditional for gather rules fail as empty sequences evaluate as "False". We need to explicitly check for "None" (the failure condition) to avoid false negatives. (cherry picked from commit e55a0e9) Co-authored-by: Pablo Galindo --- Lib/test/test_peg_generator/test_pegen.py | 16 +++++++++++++++- Tools/peg_generator/pegen/python_generator.py | 3 +++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index 5b4e964d698ade..bcfee3f2c5f8c3 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -74,7 +74,7 @@ def test_typed_rules(self) -> None: "Rule('term', 'int', Rhs([Alt([NamedItem(None, NameLeaf('NUMBER'))])]))" ) - def test_repeat_with_separator_rules(self) -> None: + def test_gather(self) -> None: grammar = """ start: ','.thing+ NEWLINE thing: NUMBER @@ -85,6 +85,20 @@ def test_repeat_with_separator_rules(self) -> None: "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" )) self.assertEqual(str(rules["thing"]), "thing: NUMBER") + parser_class = make_parser(grammar) + node = parse_string("42\n", parser_class) + assert node == [ + [[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]], + TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"), + ] + node = parse_string("1, 2\n", parser_class) + assert node == [ + [ + [TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1, 2\n")], + [TokenInfo(NUMBER, string="2", start=(1, 3), end=(1, 4), line="1, 2\n")], + ], + TokenInfo(NEWLINE, string="\n", start=(1, 4), end=(1, 5), line="1, 2\n"), + ] def test_expr_grammar(self) -> None: grammar = """ diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 45a75975dbf5e0..b786de7fee5b43 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -217,6 +217,9 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: else: self.print("and") self.visit(item) + if is_gather: + self.print("is not None") + self.print("):") with self.indent(): action = node.action