Fix typing problems reported by mypy in pegen (GH-20297)

Author: pablogsal

Tools/peg_generator/pegen/c_generator.py

@@ -1,5 +1,5 @@
 import ast
-from dataclasses import dataclass
+from dataclasses import field, dataclass
 import re
 from typing import Any, Dict, IO, Optional, List, Text, Tuple, Set
 from enum import Enum
@@ -25,6 +25,7 @@
 )
 from pegen.parser_generator import ParserGenerator
 
+
 EXTENSION_PREFIX = """\
 #include "pegen.h"
 
@@ -63,7 +64,7 @@ class NodeTypes(Enum):
 @dataclass
 class FunctionCall:
     function: str
-    arguments: Optional[List[Any]] = None
+    arguments: List[Any] = field(default_factory=list)
     assigned_variable: Optional[str] = None
     return_type: Optional[str] = None
     nodetype: Optional[NodeTypes] = None
@@ -94,7 +95,7 @@ def __init__(
         self.gen = parser_generator
         self.exact_tokens = exact_tokens
         self.non_exact_tokens = non_exact_tokens
-        self.cache: Dict[Any, Any] = {}
+        self.cache: Dict[Any, FunctionCall] = {}
         self.keyword_cache: Dict[str, int] = {}
 
     def keyword_helper(self, keyword: str) -> FunctionCall:
@@ -171,7 +172,7 @@ def can_we_inline(node: Rhs) -> int:
         if node in self.cache:
             return self.cache[node]
         if can_we_inline(node):
-            self.cache[node] = self.visit(node.alts[0].items[0])
+            self.cache[node] = self.generate_call(node.alts[0].items[0])
         else:
             name = self.gen.name_node(node)
             self.cache[node] = FunctionCall(
@@ -183,13 +184,13 @@ def can_we_inline(node: Rhs) -> int:
         return self.cache[node]
 
     def visit_NamedItem(self, node: NamedItem) -> FunctionCall:
-        call = self.visit(node.item)
+        call = self.generate_call(node.item)
         if node.name:
             call.assigned_variable = node.name
         return call
 
     def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall:
-        call = self.visit(node.node)
+        call = self.generate_call(node.node)
         if call.nodetype == NodeTypes.NAME_TOKEN:
             return FunctionCall(
                 function=f"_PyPegen_lookahead_with_name",
@@ -217,7 +218,7 @@ def visit_NegativeLookahead(self, node: NegativeLookahead) -> FunctionCall:
         return self.lookahead_call_helper(node, 0)
 
     def visit_Opt(self, node: Opt) -> FunctionCall:
-        call = self.visit(node.node)
+        call = self.generate_call(node.node)
         return FunctionCall(
             assigned_variable="_opt_var",
             function=call.function,
@@ -266,7 +267,7 @@ def visit_Gather(self, node: Gather) -> FunctionCall:
         return self.cache[node]
 
     def visit_Group(self, node: Group) -> FunctionCall:
-        return self.visit(node.rhs)
+        return self.generate_call(node.rhs)
 
     def visit_Cut(self, node: Cut) -> FunctionCall:
         return FunctionCall(
@@ -276,6 +277,9 @@ def visit_Cut(self, node: Cut) -> FunctionCall:
             nodetype=NodeTypes.CUT_OPERATOR,
         )
 
+    def generate_call(self, node: Any) -> FunctionCall:
+        return super().visit(node)
+
 
 class CParserGenerator(ParserGenerator, GrammarVisitor):
     def __init__(
@@ -317,17 +321,13 @@ def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None:
             self.print(f"goto {goto_target};")
         self.print(f"}}")
 
-    def out_of_memory_return(
-        self,
-        expr: str,
-        cleanup_code: Optional[str] = None,
-    ) -> None:
+    def out_of_memory_return(self, expr: str, cleanup_code: Optional[str] = None,) -> None:
         self.print(f"if ({expr}) {{")
         with self.indent():
             if cleanup_code is not None:
                 self.print(cleanup_code)
             self.print("p->error_indicator = 1;")
-            self.print("PyErr_NoMemory();");
+            self.print("PyErr_NoMemory();")
             self.print("return NULL;")
         self.print(f"}}")
 
@@ -484,10 +484,7 @@ def _handle_default_rule_body(self, node: Rule, rhs: Rhs, result_type: str) -> N
             if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
                 self._set_up_token_start_metadata_extraction()
             self.visit(
-                rhs,
-                is_loop=False,
-                is_gather=node.is_gather(),
-                rulename=node.name,
+                rhs, is_loop=False, is_gather=node.is_gather(), rulename=node.name,
             )
             if self.debug:
                 self.print('fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark);')
@@ -518,10 +515,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None:
             if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
                 self._set_up_token_start_metadata_extraction()
             self.visit(
-                rhs,
-                is_loop=True,
-                is_gather=node.is_gather(),
-                rulename=node.name,
+                rhs, is_loop=True, is_gather=node.is_gather(), rulename=node.name,
             )
             if is_repeat1:
                 self.print("if (_n == 0 || p->error_indicator) {")
@@ -567,7 +561,7 @@ def visit_Rule(self, node: Rule) -> None:
         self.print("}")
 
     def visit_NamedItem(self, node: NamedItem) -> None:
-        call = self.callmakervisitor.visit(node)
+        call = self.callmakervisitor.generate_call(node)
         if call.assigned_variable:
             call.assigned_variable = self.dedupe(call.assigned_variable)
         self.print(call)
@@ -674,7 +668,9 @@ def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) -
             self.print("if (_n == _children_capacity) {")
             with self.indent():
                 self.print("_children_capacity *= 2;")
-                self.print("void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));")
+                self.print(
+                    "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));"
+                )
                 self.out_of_memory_return(f"!_new_children")
                 self.print("_children = _new_children;")
             self.print("}")
@@ -721,7 +717,7 @@ def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]:
         return types
 
     def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]:
-        call = self.callmakervisitor.visit(node.item)
+        call = self.callmakervisitor.generate_call(node.item)
         name = node.name if node.name else call.assigned_variable
         if name is not None:
             name = self.dedupe(name)

Tools/peg_generator/pegen/parser_generator.py

@@ -27,7 +27,7 @@ def visit_NameLeaf(self, node: NameLeaf) -> None:
             # TODO: Add line/col info to (leaf) nodes
             raise GrammarError(f"Dangling reference to rule {node.value!r}")
 
-    def visit_NamedItem(self, node: NameLeaf) -> None:
+    def visit_NamedItem(self, node: NamedItem) -> None:
         if node.name and node.name.startswith("_"):
             raise GrammarError(f"Variable names cannot start with underscore: '{node.name}'")
         self.visit(node.item)
@@ -57,7 +57,7 @@ def __init__(self, grammar: Grammar, tokens: Dict[int, str], file: Optional[IO[T
         self.all_rules: Dict[str, Rule] = {}  # Rules + temporal rules
         self._local_variable_stack: List[List[str]] = []
 
-    def validate_rule_names(self):
+    def validate_rule_names(self) -> None:
         for rule in self.rules:
             if rule.startswith("_"):
                 raise GrammarError(f"Rule names cannot start with underscore: '{rule}'")