gh-138970: Add general metadata system to the peg generator

Author: pablogsal

Lib/test/test_peg_generator/test_pegen.py

@@ -1106,3 +1106,66 @@ def test_deep_nested_rule(self) -> None:
         )
 
         self.assertEqual(output, expected_output)
+
+    def test_rule_flags(self) -> None:
+        """Test the new rule flags syntax that accepts arbitrary lists of flags."""
+        # Test grammar with various flag combinations
+        grammar_source = """
+        start: simple_rule
+
+        simple_rule (memo):
+            | "hello"
+
+        multi_flag_rule (memo, custom, test):
+            | "world"
+
+        single_custom_flag (custom):
+            | "test"
+
+        no_flags_rule:
+            | "plain"
+        """
+
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        rules = grammar.rules
+
+        # Test memo-only rule
+        simple_rule = rules['simple_rule']
+        self.assertTrue(simple_rule.memo, "simple_rule should have memo=True")
+        self.assertEqual(simple_rule.flags, frozenset(['memo']),
+                        f"simple_rule flags should be ['memo'], got {simple_rule.flags}")
+
+        # Test multi-flag rule
+        multi_flag_rule = rules['multi_flag_rule']
+        self.assertTrue(multi_flag_rule.memo, "multi_flag_rule should have memo=True")
+        self.assertEqual(multi_flag_rule.flags, frozenset({'memo', 'custom', 'test'}),
+                        f"multi_flag_rule flags should contain memo, custom, test, got {multi_flag_rule.flags}")
+
+        # Test single custom flag rule
+        single_custom_rule = rules['single_custom_flag']
+        self.assertFalse(single_custom_rule.memo, "single_custom_flag should have memo=False")
+        self.assertEqual(single_custom_rule.flags, frozenset(['custom']),
+                        f"single_custom_flag flags should be ['custom'], got {single_custom_rule.flags}")
+
+        # Test no flags rule
+        no_flags_rule = rules['no_flags_rule']
+        self.assertFalse(no_flags_rule.memo, "no_flags_rule should have memo=False")
+        self.assertEqual(no_flags_rule.flags, [],
+                        f"no_flags_rule flags should be [], got {no_flags_rule.flags}")
+
+    def test_memo(self) -> None:
+        """Test that the old (memo) syntax works with the flag system"""
+        grammar_source = """
+        start: memo_rule
+
+        memo_rule (memo):
+            | "test"
+        """
+
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        rules = grammar.rules
+
+        memo_rule = rules['memo_rule']
+        self.assertTrue(memo_rule.memo, "memo_rule should have memo=True")
+        self.assertEqual(memo_rule.flags, frozenset(['memo']),
+                        f"memo_rule flags should be ['memo'], got {memo_rule.flags}")

Tools/peg_generator/pegen/c_generator.py

@@ -595,7 +595,7 @@ def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None:
         self.print(f"{node.name}_raw(Parser *p)")
 
     def _should_memoize(self, node: Rule) -> bool:
-        return node.memo and not node.left_recursive
+        return "memo" in node.flags and not node.left_recursive
 
     def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None:
         memoize = self._should_memoize(node)

Tools/peg_generator/pegen/grammar.py

@@ -58,11 +58,11 @@ def __iter__(self) -> Iterator[Rule]:
 
 
 class Rule:
-    def __init__(self, name: str, type: str | None, rhs: Rhs, memo: object | None = None):
+    def __init__(self, name: str, type: str | None, rhs: Rhs, flags: frozenset[str] | None = None):
         self.name = name
         self.type = type
         self.rhs = rhs
-        self.memo = bool(memo)
+        self.flags = flags or frozenset()
         self.left_recursive = False
         self.leader = False
 
@@ -135,7 +135,6 @@ def __repr__(self) -> str:
 class Rhs:
     def __init__(self, alts: list[Alt]):
         self.alts = alts
-        self.memo: tuple[str | None, str] | None = None
 
     def __str__(self) -> str:
         return " | ".join(str(alt) for alt in self.alts)
@@ -263,7 +262,6 @@ class Repeat:
 
     def __init__(self, node: Plain):
         self.node = node
-        self.memo: tuple[str | None, str] | None = None
 
     def __iter__(self) -> Iterator[Plain]:
         yield self.node

Tools/peg_generator/pegen/grammar_parser.py

@@ -50,19 +50,21 @@ rules[RuleList]:
     | rule { [rule] }
 
 rule[Rule]:
-    | rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT {
-          Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), memo=opt) }
-    | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT {
-          Rule(rulename[0], rulename[1], more_alts, memo=opt) }
-    | rulename memoflag? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, memo=opt) }
+    | rulename flags? ":" alts NEWLINE INDENT more_alts DEDENT {
+          Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), flags=opt) }
+    | rulename flags? ":" NEWLINE INDENT more_alts DEDENT {
+          Rule(rulename[0], rulename[1], more_alts, flags=opt) }
+    | rulename flags? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, flags=opt) }
 
 rulename[RuleName]:
     | NAME annotation { (name.string, annotation) }
     | NAME { (name.string, None) }
 
-# In the future this may return something more complicated
-memoflag[str]:
-    | '(' "memo" ')' { "memo" }
+flags[frozenset]:
+    | '(' a=','.flag+ ')' { frozenset(a) }
+
+flag[str]:
+    | NAME { name.string }
 
 alts[Rhs]:
     | alt "|" alts { Rhs([alt] + alts.alts)}