gh-138970: Add general metadata system to the peg generator

Author: pablogsal

Lib/test/test_peg_generator/test_pegen.py

@@ -1106,3 +1106,66 @@ def test_deep_nested_rule(self) -> None:
         )
 
         self.assertEqual(output, expected_output)
+
+    def test_rule_flags(self) -> None:
+        """Test the new rule flags syntax that accepts arbitrary lists of flags."""
+        # Test grammar with various flag combinations
+        grammar_source = """
+        start: simple_rule
+
+        simple_rule (memo):
+            | "hello"
+
+        multi_flag_rule (memo, custom, test):
+            | "world"
+
+        single_custom_flag (custom):
+            | "test"
+
+        no_flags_rule:
+            | "plain"
+        """
+
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        rules = grammar.rules
+
+        # Test memo-only rule
+        simple_rule = rules['simple_rule']
+        self.assertTrue(simple_rule.memo, "simple_rule should have memo=True")
+        self.assertEqual(simple_rule.flags, frozenset(['memo']),
+                        f"simple_rule flags should be ['memo'], got {simple_rule.flags}")
+
+        # Test multi-flag rule
+        multi_flag_rule = rules['multi_flag_rule']
+        self.assertTrue(multi_flag_rule.memo, "multi_flag_rule should have memo=True")
+        self.assertEqual(multi_flag_rule.flags, frozenset({'memo', 'custom', 'test'}),
+                        f"multi_flag_rule flags should contain memo, custom, test, got {multi_flag_rule.flags}")
+
+        # Test single custom flag rule
+        single_custom_rule = rules['single_custom_flag']
+        self.assertFalse(single_custom_rule.memo, "single_custom_flag should have memo=False")
+        self.assertEqual(single_custom_rule.flags, frozenset(['custom']),
+                        f"single_custom_flag flags should be ['custom'], got {single_custom_rule.flags}")
+
+        # Test no flags rule
+        no_flags_rule = rules['no_flags_rule']
+        self.assertFalse(no_flags_rule.memo, "no_flags_rule should have memo=False")
+        self.assertEqual(no_flags_rule.flags, [],
+                        f"no_flags_rule flags should be [], got {no_flags_rule.flags}")
+
+    def test_memo(self) -> None:
+        """Test that the old (memo) syntax works with the flag system"""
+        grammar_source = """
+        start: memo_rule
+
+        memo_rule (memo):
+            | "test"
+        """
+
+        grammar: Grammar = parse_string(grammar_source, GrammarParser)
+        rules = grammar.rules
+
+        memo_rule = rules['memo_rule']
+        self.assertTrue(memo_rule.memo, "memo_rule should have memo=True")
+        self.assertEqual(memo_rule.flags, frozenset(['memo']),
+                        f"memo_rule flags should be ['memo'], got {memo_rule.flags}")

Tools/peg_generator/pegen/grammar.py

@@ -58,14 +58,19 @@ def __iter__(self) -> Iterator[Rule]:
 
 
 class Rule:
-    def __init__(self, name: str, type: str | None, rhs: Rhs, memo: object | None = None):
+    def __init__(self, name: str, type: str | None, rhs: Rhs, memo: object | None = None, flags: list[str] | None = None):
         self.name = name
         self.type = type
         self.rhs = rhs
-        self.memo = bool(memo)
+        self.flags = flags or []
         self.left_recursive = False
         self.leader = False
 
+    @property
+    def memo(self) -> bool:
+        """Check if this rule should be memoized by looking for 'memo' in flags."""
+        return "memo" in self.flags
+
     def is_loop(self) -> bool:
         return self.name.startswith("_loop")
 

Tools/peg_generator/pegen/grammar_parser.py

@@ -50,19 +50,21 @@ rules[RuleList]:
     | rule { [rule] }
 
 rule[Rule]:
-    | rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT {
-          Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), memo=opt) }
-    | rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT {
-          Rule(rulename[0], rulename[1], more_alts, memo=opt) }
-    | rulename memoflag? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, memo=opt) }
+    | rulename flags? ":" alts NEWLINE INDENT more_alts DEDENT {
+          Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), flags=opt) }
+    | rulename flags? ":" NEWLINE INDENT more_alts DEDENT {
+          Rule(rulename[0], rulename[1], more_alts, flags=opt) }
+    | rulename flags? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, flags=opt) }
 
 rulename[RuleName]:
     | NAME annotation { (name.string, annotation) }
     | NAME { (name.string, None) }
 
-# In the future this may return something more complicated
-memoflag[str]:
-    | '(' "memo" ')' { "memo" }
+flags[list]:
+    | '(' a=','.flag+ ')' { frozenset(a) }
+
+flag[str]:
+    | NAME { name.string }
 
 alts[Rhs]:
     | alt "|" alts { Rhs([alt] + alts.alts)}

Tools/peg_generator/pegen/metagrammar.gram

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+
+import sys
+import tempfile
+import os
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'pegen'))
+
+from pegen.build import build_python_parser_and_generator
+
+def test_new_flags_syntax():
+    """Test that the new flags syntax works correctly"""
+
+    # Test grammar with various flag combinations
+    test_grammar = '''
+    start[str]: simple_rule
+
+    simple_rule (memo):
+        | "hello" { "memo_only" }
+
+    multi_flag_rule (memo, custom, test):
+        | "world" { "multiple_flags" }
+
+    single_custom_flag (custom):
+        | "test" { "single_custom" }
+
+    no_flags_rule:
+        | "plain" { "no_flags" }
+    '''
+
+    # Write test grammar to temporary file
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.gram', delete=False) as f:
+        f.write(test_grammar)
+        grammar_file = f.name
+
+    try:
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
+            output_file = f.name
+
+        # Parse the grammar
+        grammar, parser, tokenizer, gen = build_python_parser_and_generator(
+            grammar_file, output_file, verbose_parser=True
+        )
+
+        # Check that rules have correct flags
+        rules = grammar.rules
+
+        # Test memo-only rule
+        simple_rule = rules['simple_rule']
+        assert simple_rule.memo == True, "simple_rule should have memo=True"
+        assert simple_rule.flags == ['memo'], f"simple_rule flags should be ['memo'], got {simple_rule.flags}"
+
+        # Test multi-flag rule
+        multi_flag_rule = rules['multi_flag_rule']
+        assert multi_flag_rule.memo == True, "multi_flag_rule should have memo=True"
+        assert set(multi_flag_rule.flags) == {'memo', 'custom', 'test'}, f"multi_flag_rule flags should contain memo, custom, test, got {multi_flag_rule.flags}"
+
+        # Test single custom flag rule
+        single_custom_rule = rules['single_custom_flag']
+        assert single_custom_rule.memo == False, "single_custom_flag should have memo=False"
+        assert single_custom_rule.flags == ['custom'], f"single_custom_flag flags should be ['custom'], got {single_custom_rule.flags}"
+
+        # Test no flags rule
+        no_flags_rule = rules['no_flags_rule']
+        assert no_flags_rule.memo == False, "no_flags_rule should have memo=False"
+        assert no_flags_rule.flags == [], f"no_flags_rule flags should be [], got {no_flags_rule.flags}"
+
+        print("✅ All tests passed!")
+        print(f"✅ simple_rule: memo={simple_rule.memo}, flags={simple_rule.flags}")
+        print(f"✅ multi_flag_rule: memo={multi_flag_rule.memo}, flags={multi_flag_rule.flags}")
+        print(f"✅ single_custom_flag: memo={single_custom_rule.memo}, flags={single_custom_rule.flags}")
+        print(f"✅ no_flags_rule: memo={no_flags_rule.memo}, flags={no_flags_rule.flags}")
+
+    finally:
+        # Clean up
+        os.unlink(grammar_file)
+        if os.path.exists(output_file):
+            os.unlink(output_file)
+
+if __name__ == '__main__':
+    test_new_flags_syntax()