Merge pull request #64 from robotpy/state-user-data

 Allow parser visitors to leverage the parser's state instead of creating their own stacks

Author: virtuald

cxxheaderparser/parser.py

@@ -106,6 +106,8 @@ def debug_print(fmt: str, *args: typing.Any) -> None:
         else:
             self.debug_print = lambda fmt, *args: None
 
+        self.visitor.on_parse_start(self.state)
+
     #
     # State management
     #
@@ -505,7 +507,8 @@ def _consume_static_assert(
     def _on_empty_block_start(
         self, tok: LexToken, doxygen: typing.Optional[str]
     ) -> None:
-        self._push_state(EmptyBlockState)
+        state = self._push_state(EmptyBlockState)
+        self.visitor.on_empty_block_start(state)
 
     def _on_block_end(self, tok: LexToken, doxygen: typing.Optional[str]) -> None:
         old_state = self._pop_state()

cxxheaderparser/parserstate.py

@@ -28,26 +28,41 @@ def validate(self, *, var_ok: bool, meth_ok: bool, msg: str) -> None:
                 raise CxxParseError(f"{msg}: unexpected '{tok.value}'")
 
 
-class State:
+#: custom user data for this state type
+T = typing.TypeVar("T")
+
+#: type of custom user data for a parent state
+PT = typing.TypeVar("PT")
+
+
+class State(typing.Generic[T, PT]):
+    #: Uninitialized user data available for use by visitor implementations. You
+    #: should set this in a ``*_start`` method.
+    user_data: T
+
     #: parent state
-    parent: typing.Optional["State"]
+    parent: typing.Optional["State[PT, typing.Any]"]
 
     #: Approximate location that the parsed element was found at
     location: Location
 
-    def __init__(self, parent: typing.Optional["State"]) -> None:
+    def __init__(self, parent: typing.Optional["State[PT, typing.Any]"]) -> None:
         self.parent = parent
 
     def _finish(self, visitor: "CxxVisitor") -> None:
         pass
 
 
-class EmptyBlockState(State):
+class EmptyBlockState(State[T, PT]):
+    parent: State[PT, typing.Any]
+
     def _finish(self, visitor: "CxxVisitor") -> None:
         visitor.on_empty_block_end(self)
 
 
-class ExternBlockState(State):
+class ExternBlockState(State[T, PT]):
+    parent: State[PT, typing.Any]
+
     #: The linkage for this extern block
     linkage: str
 
@@ -59,7 +74,9 @@ def _finish(self, visitor: "CxxVisitor") -> None:
         visitor.on_extern_block_end(self)
 
 
-class NamespaceBlockState(State):
+class NamespaceBlockState(State[T, PT]):
+    parent: State[PT, typing.Any]
+
     #: The incremental namespace for this block
     namespace: NamespaceDecl
 
@@ -73,7 +90,9 @@ def _finish(self, visitor: "CxxVisitor") -> None:
         visitor.on_namespace_end(self)
 
 
-class ClassBlockState(State):
+class ClassBlockState(State[T, PT]):
+    parent: State[PT, typing.Any]
+
     #: class decl block being processed
     class_decl: ClassDecl
 

cxxheaderparser/simple.py

@@ -179,6 +179,13 @@ class N::C {
 # Visitor implementation
 #
 
+# define what user data we store in each state type
+SState = State[Block, Block]
+SEmptyBlockState = EmptyBlockState[Block, Block]
+SExternBlockState = ExternBlockState[Block, Block]
+SNamespaceBlockState = NamespaceBlockState[NamespaceScope, NamespaceScope]
+SClassBlockState = ClassBlockState[ClassScope, Block]
+
 
 class SimpleCxxVisitor:
     """
@@ -190,43 +197,35 @@ class SimpleCxxVisitor:
     """
 
     data: ParsedData
-    namespace: NamespaceScope
-    block: Block
-
-    def __init__(self) -> None:
-        self.namespace = NamespaceScope("")
-        self.block = self.namespace
-
-        self.ns_stack = typing.Deque[NamespaceScope]()
-        self.block_stack = typing.Deque[Block]()
 
-        self.data = ParsedData(self.namespace)
+    def on_parse_start(self, state: SNamespaceBlockState) -> None:
+        ns = NamespaceScope("")
+        self.data = ParsedData(ns)
+        state.user_data = ns
 
-    def on_pragma(self, state: State, content: Value) -> None:
+    def on_pragma(self, state: SState, content: Value) -> None:
         self.data.pragmas.append(Pragma(content))
 
-    def on_include(self, state: State, filename: str) -> None:
+    def on_include(self, state: SState, filename: str) -> None:
         self.data.includes.append(Include(filename))
 
-    def on_empty_block_start(self, state: EmptyBlockState) -> None:
+    def on_empty_block_start(self, state: SEmptyBlockState) -> None:
         # this matters for some scope/resolving purposes, but you're
         # probably going to want to use clang if you care about that
         # level of detail
-        pass
+        state.user_data = state.parent.user_data
 
-    def on_empty_block_end(self, state: EmptyBlockState) -> None:
+    def on_empty_block_end(self, state: SEmptyBlockState) -> None:
         pass
 
-    def on_extern_block_start(self, state: ExternBlockState) -> None:
-        pass  # TODO
+    def on_extern_block_start(self, state: SExternBlockState) -> None:
+        state.user_data = state.parent.user_data
 
-    def on_extern_block_end(self, state: ExternBlockState) -> None:
+    def on_extern_block_end(self, state: SExternBlockState) -> None:
         pass
 
-    def on_namespace_start(self, state: NamespaceBlockState) -> None:
-        parent_ns = self.namespace
-        self.block_stack.append(parent_ns)
-        self.ns_stack.append(parent_ns)
+    def on_namespace_start(self, state: SNamespaceBlockState) -> None:
+        parent_ns = state.parent.user_data
 
         ns = None
         names = state.namespace.names
@@ -247,81 +246,76 @@ def on_namespace_start(self, state: NamespaceBlockState) -> None:
         ns.inline = state.namespace.inline
         ns.doxygen = state.namespace.doxygen
 
-        self.block = ns
-        self.namespace = ns
+        state.user_data = ns
 
-    def on_namespace_end(self, state: NamespaceBlockState) -> None:
-        self.block = self.block_stack.pop()
-        self.namespace = self.ns_stack.pop()
+    def on_namespace_end(self, state: SNamespaceBlockState) -> None:
+        pass
 
-    def on_namespace_alias(self, state: State, alias: NamespaceAlias) -> None:
-        assert isinstance(self.block, NamespaceScope)
-        self.block.ns_alias.append(alias)
+    def on_namespace_alias(self, state: SState, alias: NamespaceAlias) -> None:
+        assert isinstance(state.user_data, NamespaceScope)
+        state.user_data.ns_alias.append(alias)
 
-    def on_forward_decl(self, state: State, fdecl: ForwardDecl) -> None:
-        self.block.forward_decls.append(fdecl)
+    def on_forward_decl(self, state: SState, fdecl: ForwardDecl) -> None:
+        state.user_data.forward_decls.append(fdecl)
 
-    def on_template_inst(self, state: State, inst: TemplateInst) -> None:
-        assert isinstance(self.block, NamespaceScope)
-        self.block.template_insts.append(inst)
+    def on_template_inst(self, state: SState, inst: TemplateInst) -> None:
+        assert isinstance(state.user_data, NamespaceScope)
+        state.user_data.template_insts.append(inst)
 
-    def on_variable(self, state: State, v: Variable) -> None:
-        assert isinstance(self.block, NamespaceScope)
-        self.block.variables.append(v)
+    def on_variable(self, state: SState, v: Variable) -> None:
+        assert isinstance(state.user_data, NamespaceScope)
+        state.user_data.variables.append(v)
 
-    def on_function(self, state: State, fn: Function) -> None:
-        assert isinstance(self.block, NamespaceScope)
-        self.block.functions.append(fn)
+    def on_function(self, state: SState, fn: Function) -> None:
+        assert isinstance(state.user_data, NamespaceScope)
+        state.user_data.functions.append(fn)
 
-    def on_method_impl(self, state: State, method: Method) -> None:
-        assert isinstance(self.block, NamespaceScope)
-        self.block.method_impls.append(method)
+    def on_method_impl(self, state: SState, method: Method) -> None:
+        assert isinstance(state.user_data, NamespaceScope)
+        state.user_data.method_impls.append(method)
 
-    def on_typedef(self, state: State, typedef: Typedef) -> None:
-        self.block.typedefs.append(typedef)
+    def on_typedef(self, state: SState, typedef: Typedef) -> None:
+        state.user_data.typedefs.append(typedef)
 
-    def on_using_namespace(self, state: State, namespace: typing.List[str]) -> None:
-        assert isinstance(self.block, NamespaceScope)
+    def on_using_namespace(self, state: SState, namespace: typing.List[str]) -> None:
+        assert isinstance(state.user_data, NamespaceScope)
         ns = UsingNamespace("::".join(namespace))
-        self.block.using_ns.append(ns)
+        state.user_data.using_ns.append(ns)
 
-    def on_using_alias(self, state: State, using: UsingAlias) -> None:
-        self.block.using_alias.append(using)
+    def on_using_alias(self, state: SState, using: UsingAlias) -> None:
+        state.user_data.using_alias.append(using)
 
-    def on_using_declaration(self, state: State, using: UsingDecl) -> None:
-        self.block.using.append(using)
+    def on_using_declaration(self, state: SState, using: UsingDecl) -> None:
+        state.user_data.using.append(using)
 
     #
     # Enums
     #
 
-    def on_enum(self, state: State, enum: EnumDecl) -> None:
-        self.block.enums.append(enum)
+    def on_enum(self, state: SState, enum: EnumDecl) -> None:
+        state.user_data.enums.append(enum)
 
     #
     # Class/union/struct
     #
 
-    def on_class_start(self, state: ClassBlockState) -> None:
+    def on_class_start(self, state: SClassBlockState) -> None:
+        parent = state.parent.user_data
         block = ClassScope(state.class_decl)
-        self.block.classes.append(block)
-        self.block_stack.append(self.block)
-        self.block = block
+        parent.classes.append(block)
+        state.user_data = block
 
-    def on_class_field(self, state: State, f: Field) -> None:
-        assert isinstance(self.block, ClassScope)
-        self.block.fields.append(f)
+    def on_class_field(self, state: SClassBlockState, f: Field) -> None:
+        state.user_data.fields.append(f)
 
-    def on_class_method(self, state: ClassBlockState, method: Method) -> None:
-        assert isinstance(self.block, ClassScope)
-        self.block.methods.append(method)
+    def on_class_method(self, state: SClassBlockState, method: Method) -> None:
+        state.user_data.methods.append(method)
 
-    def on_class_friend(self, state: ClassBlockState, friend: FriendDecl) -> None:
-        assert isinstance(self.block, ClassScope)
-        self.block.friends.append(friend)
+    def on_class_friend(self, state: SClassBlockState, friend: FriendDecl) -> None:
+        state.user_data.friends.append(friend)
 
-    def on_class_end(self, state: ClassBlockState) -> None:
-        self.block = self.block_stack.pop()
+    def on_class_end(self, state: SClassBlockState) -> None:
+        pass
 
 
 def parse_string(

cxxheaderparser/visitor.py

@@ -37,6 +37,11 @@ class CxxVisitor(Protocol):
     Defines the interface used by the parser to emit events
     """
 
+    def on_parse_start(self, state: NamespaceBlockState) -> None:
+        """
+        Called when parsing begins
+        """
+
     def on_pragma(self, state: State, content: Value) -> None:
         """
         Called once for each ``#pragma`` directive encountered

tests/test_misc.py

@@ -351,3 +351,29 @@ def test_warning_directive() -> None:
     data = parse_string(content, cleandoc=True)
 
     assert data == ParsedData()
+
+
+def test_empty_block() -> None:
+    """
+    Ensure the simple visitor doesn't break with an empty block
+    """
+    content = """
+      {
+          class X {};
+      }
+    """
+    data = parse_string(content, cleandoc=True)
+
+    assert data == ParsedData(
+        namespace=NamespaceScope(
+            classes=[
+                ClassScope(
+                    class_decl=ClassDecl(
+                        typename=PQName(
+                            segments=[NameSpecifier(name="X")], classkey="class"
+                        )
+                    )
+                )
+            ]
+        )
+    )