gh-138970: Add general metadata system to the peg generator

Author: pablogsal

Lib/test/test_peg_generator/test_pegen.py

@@ -1106,3 +1106,66 @@ def test_deep_nested_rule(self) -> None:
         )
 
         self.assertEqual(output, expected_output)
+
+    def test_rule_flags(self) -> None:
+        """Test the new rule flags syntax that accepts arbitrary lists of flags."""
+        # Test grammar with various flag combinations
+        grammar_source = """
+        start: simple_rule
+
+        simple_rule (memo):
+            | "hello"
+
+        multi_flag_rule (memo, custom, test):
+            | "world"
+
+        single_custom_flag (custom):
+            | "test"
+
+        no_flags_rule:
+            | "plain"
+        """
+
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        rules = grammar.rules
+
+        # Test memo-only rule
+        simple_rule = rules['simple_rule']
+        self.assertTrue(simple_rule.memo, "simple_rule should have memo=True")
+        self.assertEqual(simple_rule.flags, frozenset(['memo']),
+                        f"simple_rule flags should be ['memo'], got {simple_rule.flags}")
+
+        # Test multi-flag rule
+        multi_flag_rule = rules['multi_flag_rule']
+        self.assertTrue(multi_flag_rule.memo, "multi_flag_rule should have memo=True")
+        self.assertEqual(multi_flag_rule.flags, frozenset({'memo', 'custom', 'test'}),
+                        f"multi_flag_rule flags should contain memo, custom, test, got {multi_flag_rule.flags}")
+
+        # Test single custom flag rule
+        single_custom_rule = rules['single_custom_flag']
+        self.assertFalse(single_custom_rule.memo, "single_custom_flag should have memo=False")
+        self.assertEqual(single_custom_rule.flags, frozenset(['custom']),
+                        f"single_custom_flag flags should be ['custom'], got {single_custom_rule.flags}")
+
+        # Test no flags rule
+        no_flags_rule = rules['no_flags_rule']
+        self.assertFalse(no_flags_rule.memo, "no_flags_rule should have memo=False")
+        self.assertEqual(no_flags_rule.flags, [],
+                        f"no_flags_rule flags should be [], got {no_flags_rule.flags}")
+
+    def test_memo(self) -> None:
+        """Test that the old (memo) syntax works with the flag system"""
+        grammar_source = """
+        start: memo_rule
+
+        memo_rule (memo):
+            | "test"
+        """
+
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        rules = grammar.rules
+
+        memo_rule = rules['memo_rule']
+        self.assertTrue(memo_rule.memo, "memo_rule should have memo=True")
+        self.assertEqual(memo_rule.flags, frozenset(['memo']),
+                        f"memo_rule flags should be ['memo'], got {memo_rule.flags}")

Tools/peg_generator/pegen/grammar.py

@@ -58,14 +58,19 @@ def __iter__(self) -> Iterator[Rule]:
 
 
 class Rule:
-    def __init__(self, name: str, type: str | None, rhs: Rhs, memo: object | None = None):
+    def __init__(self, name: str, type: str | None, rhs: Rhs, memo: object | None = None, flags: list[str] | None = None):
         self.name = name
         self.type = type
         self.rhs = rhs
-        self.memo = bool(memo)
+        self.flags = flags or []
         self.left_recursive = False
         self.leader = False
 
+    @property
+    def memo(self) -> bool:
+        """Check if this rule should be memoized by looking for 'memo' in flags."""
+        return "memo" in self.flags
+
     def is_loop(self) -> bool:
         return self.name.startswith("_loop")
 

Tools/peg_generator/pegen/grammar_parser.py

@@ -50,19 +50,21 @@ rules[RuleList]:
     | rule { [rule] }
 
 rule[Rule]:
-    | rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT {
-          Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), memo=opt) }
-    | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT {
-          Rule(rulename[0], rulename[1], more_alts, memo=opt) }
-    | rulename memoflag? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, memo=opt) }
+    | rulename flags? ":" alts NEWLINE INDENT more_alts DEDENT {
+          Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), flags=opt) }
+    | rulename flags? ":" NEWLINE INDENT more_alts DEDENT {
+          Rule(rulename[0], rulename[1], more_alts, flags=opt) }
+    | rulename flags? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, flags=opt) }
 
 rulename[RuleName]:
     | NAME annotation { (name.string, annotation) }
     | NAME { (name.string, None) }
 
-# In the future this may return something more complicated
-memoflag[str]:
-    | '(' "memo" ')' { "memo" }
+flags[list]:
+    | '(' a=','.flag+ ')' { frozenset(a) }
+
+flag[str]:
+    | NAME { name.string }
 
 alts[Rhs]:
     | alt "|" alts { Rhs([alt] + alts.alts)}