Skip to content

Commit 0ce9fb7

Browse files
authored
gh-138970: Add general metadata system to the peg generator (#138971)
1 parent b485e50 commit 0ce9fb7

File tree

5 files changed

+113
-26
lines changed

5 files changed

+113
-26
lines changed

Lib/test/test_peg_generator/test_pegen.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1106,3 +1106,49 @@ def test_deep_nested_rule(self) -> None:
11061106
)
11071107

11081108
self.assertEqual(output, expected_output)
1109+
1110+
def test_rule_flags(self) -> None:
1111+
"""Test the new rule flags syntax that accepts arbitrary lists of flags."""
1112+
# Test grammar with various flag combinations
1113+
grammar_source = """
1114+
start: simple_rule
1115+
1116+
simple_rule (memo):
1117+
| "hello"
1118+
1119+
multi_flag_rule (memo, custom, test):
1120+
| "world"
1121+
1122+
single_custom_flag (custom):
1123+
| "test"
1124+
1125+
no_flags_rule:
1126+
| "plain"
1127+
"""
1128+
1129+
grammar: Grammar = parse_string(grammar_source, GrammarParser)
1130+
rules = grammar.rules
1131+
1132+
# Test memo-only rule
1133+
simple_rule = rules['simple_rule']
1134+
self.assertTrue(simple_rule.memo, "simple_rule should have memo=True")
1135+
self.assertEqual(simple_rule.flags, frozenset(['memo']),
1136+
f"simple_rule flags should be {'memo'}, got {simple_rule.flags}")
1137+
1138+
# Test multi-flag rule
1139+
multi_flag_rule = rules['multi_flag_rule']
1140+
self.assertTrue(multi_flag_rule.memo, "multi_flag_rule should have memo=True")
1141+
self.assertEqual(multi_flag_rule.flags, frozenset({'memo', 'custom', 'test'}),
1142+
f"multi_flag_rule flags should contain memo, custom, test, got {multi_flag_rule.flags}")
1143+
1144+
# Test single custom flag rule
1145+
single_custom_rule = rules['single_custom_flag']
1146+
self.assertFalse(single_custom_rule.memo, "single_custom_flag should have memo=False")
1147+
self.assertEqual(single_custom_rule.flags, frozenset(['custom']),
1148+
f"single_custom_flag flags should be {'custom'}, got {single_custom_rule.flags}")
1149+
1150+
# Test no flags rule
1151+
no_flags_rule = rules['no_flags_rule']
1152+
self.assertFalse(no_flags_rule.memo, "no_flags_rule should have memo=False")
1153+
self.assertEqual(no_flags_rule.flags, [],
1154+
f"no_flags_rule flags should be the empty set, got {no_flags_rule.flags}")

Tools/peg_generator/pegen/c_generator.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -595,7 +595,7 @@ def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None:
595595
self.print(f"{node.name}_raw(Parser *p)")
596596

597597
def _should_memoize(self, node: Rule) -> bool:
598-
return node.memo and not node.left_recursive
598+
return "memo" in node.flags and not node.left_recursive
599599

600600
def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> None:
601601
memoize = self._should_memoize(node)

Tools/peg_generator/pegen/grammar.py

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,11 @@ def __iter__(self) -> Iterator[Rule]:
5858

5959

6060
class Rule:
61-
def __init__(self, name: str, type: str | None, rhs: Rhs, memo: object | None = None):
61+
def __init__(self, name: str, type: str | None, rhs: Rhs, flags: frozenset[str] | None = None):
6262
self.name = name
6363
self.type = type
6464
self.rhs = rhs
65-
self.memo = bool(memo)
65+
self.flags = flags or frozenset()
6666
self.left_recursive = False
6767
self.leader = False
6868

@@ -135,7 +135,6 @@ def __repr__(self) -> str:
135135
class Rhs:
136136
def __init__(self, alts: list[Alt]):
137137
self.alts = alts
138-
self.memo: tuple[str | None, str] | None = None
139138

140139
def __str__(self) -> str:
141140
return " | ".join(str(alt) for alt in self.alts)
@@ -263,7 +262,6 @@ class Repeat:
263262

264263
def __init__(self, node: Plain):
265264
self.node = node
266-
self.memo: tuple[str | None, str] | None = None
267265

268266
def __iter__(self) -> Iterator[Plain]:
269267
yield self.node

Tools/peg_generator/pegen/grammar_parser.py

Lines changed: 54 additions & 13 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Tools/peg_generator/pegen/metagrammar.gram

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,19 +50,21 @@ rules[RuleList]:
5050
| rule { [rule] }
5151

5252
rule[Rule]:
53-
| rulename memoflag? ":" alts NEWLINE INDENT more_alts DEDENT {
54-
Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), memo=opt) }
55-
| rulename memoflag? ":" NEWLINE INDENT more_alts DEDENT {
56-
Rule(rulename[0], rulename[1], more_alts, memo=opt) }
57-
| rulename memoflag? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, memo=opt) }
53+
| rulename flags=flags? ":" alts NEWLINE INDENT more_alts DEDENT {
54+
Rule(rulename[0], rulename[1], Rhs(alts.alts + more_alts.alts), flags=flags) }
55+
| rulename flags=flags? ":" NEWLINE INDENT more_alts DEDENT {
56+
Rule(rulename[0], rulename[1], more_alts, flags=flags) }
57+
| rulename flags=flags? ":" alts NEWLINE { Rule(rulename[0], rulename[1], alts, flags=flags) }
5858

5959
rulename[RuleName]:
6060
| NAME annotation { (name.string, annotation) }
6161
| NAME { (name.string, None) }
6262

63-
# In the future this may return something more complicated
64-
memoflag[str]:
65-
| '(' "memo" ')' { "memo" }
63+
flags[frozenset[str]]:
64+
| '(' a=','.flag+ ')' { frozenset(a) }
65+
66+
flag[str]:
67+
| NAME { name.string }
6668

6769
alts[Rhs]:
6870
| alt "|" alts { Rhs([alt] + alts.alts)}

0 commit comments

Comments
 (0)