Merge pull request #64 from hit9/dev1

 Editor: A simple language server for bitproto

Author: hit9

changes.rst

@@ -1,32 +1,40 @@
 .. currentmodule:: bitproto
 
+Version 1.2.0
+-------------
+
+.. _version-1.2.0:
+
+- Feature: Add a simple language-server, tested on neovim and vscode.
+- Editor:  Upgrade vscode extenions to support the ``bitproto-language-server``.
+
 Version 1.1.2
 -------------
 
-.. _version-1.1.2
+.. _version-1.1.2:
 
 - Feature: Allow using constants as option values. ISSUE #61, PR #63
 
 Version 1.1.1
 -------------
 
-.. _version-1.1.1
+.. _version-1.1.1:
 
 - Fix bug: enum importing other bitproto's field name generation bug. #53 #52
 - Fix bug: import statements of bitprotos should be placed ahead of other declarations. #53
 
 Version 1.1.0
 -------------
 
-.. _version-1.1.0
+.. _version-1.1.0:
 
 - Performance improvements for C bitprotolib, 40~60us improvement per call on stm32. PR #48.
 - Fix Python nested message ``__post_init___`` function code generation. PR #48, commit 73f4b01.
 
 Version 1.0.1
 -------------
 
-.. _version-1.0.1
+.. _version-1.0.1:
 
 - Add support for Python 3.11
 

compiler/MANIFEST.in

@@ -7,3 +7,4 @@ recursive-exclude *.egg-info *
 recursive-exclude .git *
 recursive-exclude .mypy_cache *
 recursive-exclude __pycache__ *
+recursive-exclude .ruff_cache *

compiler/bitproto/__init__.py

@@ -8,5 +8,5 @@
 
 """
 
-__version__ = "1.1.2"
+__version__ = "1.2.0"
 __description__ = "bit level data interchange format."

compiler/bitproto/_ast.py

@@ -39,6 +39,7 @@
       |    |    |- Enum                 :Scope:Definition:Node
       |    |    |- Message              :Scope:Definition:Node
       |    |    |- Proto                :Scope:Definition:Node
+      |- Reference                      :Node
 """
 
 from collections import OrderedDict as dict_
@@ -130,11 +131,18 @@ def cache_if_frozen_condition(
 class Node:
     token: str = ""
     lineno: int = 0
+    # column start and end of this token in the line, starting from 0.
+    # to get the col end: col_start + len(token)
+    token_col_start: int = 0
     filepath: str = ""
 
     # cheat mypy: override by @frozen
     __frozen__: ClassVar[bool] = False
 
+    @property
+    def token_col_end(self) -> int:
+        return self.token_col_start + len(self.token)
+
     def is_frozen(self) -> bool:
         return self.__frozen__
 
@@ -192,6 +200,18 @@ def __repr__(self) -> str:
         return f"<{class_repr} {self.name}>"
 
 
+@dataclass
+class Reference(Node):
+    """
+    Reference to a definition.
+    """
+
+    referenced_definition: Optional[Definition] = None
+
+    def __repr__(self) -> str:
+        return f"<{self.token}>"
+
+
 @dataclass
 class BoundDefinition(Definition):
     """BoundDefinition is definitions bound to a proto,
@@ -368,6 +388,12 @@ class Scope(Definition):
 
     members: "dict_[str, Definition]" = dataclass_field(default_factory=dict_)
 
+    # scope's start and end (normally, the '{' and '}' positions)
+    scope_start_lineno: int = 0
+    scope_start_col: int = 0
+    scope_end_lineno: int = 0
+    scope_end_col: int = 0
+
     def push_member(self, member: Definition, name: Optional[str] = None) -> None:
         """Push a definition `member`, and run hook functions around.
         Raises error if given member's name already taken by another member.
@@ -1037,6 +1063,8 @@ def validate_message_field_on_push(self, field: MessageField) -> None:
 class Proto(ScopeWithOptions):
     __option_descriptors__: ClassVar[OptionDescriptors] = PROTO_OPTTIONS
 
+    references: List[Reference] = dataclass_field(default_factory=list)
+
     def set_name(self, name: str) -> None:
         if self.is_frozen():
             raise InternalError("set_name on frozen proto")

compiler/bitproto/parser.py

@@ -26,6 +26,7 @@
     MessageField,
     Option,
     Proto,
+    Reference,
     Scope,
     Type,
 )
@@ -189,12 +190,9 @@ def util_parse_sequence(self, p: P) -> None:
             p[0] = []
 
     def copy_p_tracking(self, p: P, from_: int = 1, to: int = 0) -> None:
-        """Don't know why P's tracking info (lexpos and lineno) sometimes missing.
-        Particular in recursion grammar situation. We have to copy it manually.
-
-        Add this function in a p_xxx function when:
-            1. the p[0] is gona to be used in another parsing target.
-            2. and the tracking information is gona to be used there.
+        """
+        Ply's position tracking works only for lexing SYMBOLS (not for all grammer symbols) by default.
+        We either enable parse(tracking=True), or copy them on need manually.
         """
         p.set_lexpos(to, p.lexpos(from_))
         p.set_lineno(to, p.lineno(from_))
@@ -213,13 +211,20 @@ def p_open_global_scope(self, p: P) -> None:
             filepath=self.current_filepath(),
             _bound=None,
             scope_stack=self.current_scope_stack(),
+            scope_start_lineno=1,
+            scope_start_col=1,
         )
         self.push_scope(proto)
 
     @override_docstring(r_close_global_scope)
     def p_close_global_scope(self, p: P) -> None:
         scope = self.pop_scope()
         proto = cast_or_raise(Proto, scope)
+
+        proto.scope_end_lineno = p.lexer.lexdata.count("\n")  # FIXME: slow?
+        lexpos = len(p.lexer.lexdata)
+        proto.scope_end_col = lexpos - p.lexer.lexdata.rfind("\n", 0, lexpos)
+
         if not proto.name:
             raise ProtoNameUndefined(filepath=self.current_filepath())
         proto.freeze()
@@ -334,6 +339,7 @@ def p_option(self, p: P) -> None:
             filepath=self.current_filepath(),
             lineno=p.lineno(2),
             token=p[2],
+            token_col_start=self._get_col(p, 2),
         )
         self.current_scope().push_member(option)
 
@@ -357,6 +363,7 @@ def p_alias(self, p: P) -> None:
             filepath=self.current_filepath(),
             lineno=lineno,
             token=token,
+            token_col_start=self._get_col(p, 2) if len(p) == 6 else self._get_col(p, 3),
             indent=self.current_indent(p),
             scope_stack=self.current_scope_stack(),
             comment_block=self.collect_comment_block(),
@@ -384,6 +391,7 @@ def p_const(self, p: P) -> None:
             _bound=self.current_proto(),
             filepath=self.current_filepath(),
             token=p[2],
+            token_col_start=self._get_col(p, 2),
             lineno=p.lineno(2),
         )
         self.current_scope().push_member(constant)
@@ -466,6 +474,15 @@ def p_constant_reference(self, p: P) -> None:
         p[0] = d
         self.copy_p_tracking(p)
 
+        reference = Reference(
+            token=p[1],
+            lineno=p.lineno(1),
+            token_col_start=self._get_col(p, 1),
+            filepath=self.current_filepath(),
+            referenced_definition=d,
+        )
+        self.current_proto().references.append(reference)
+
     @override_docstring(r_type)
     def p_type(self, p: P) -> None:
         p[0] = p[1]
@@ -498,9 +515,19 @@ def p_type_reference(self, p: P) -> None:
                 token=p[1],
                 lineno=p.lineno(1),
             )
+
         p[0] = d
         self.copy_p_tracking(p)
 
+        reference = Reference(
+            token=p[1],
+            lineno=p.lineno(1),
+            token_col_start=self._get_col(p, 1),
+            filepath=self.current_filepath(),
+            referenced_definition=d,
+        )
+        self.current_proto().references.append(reference)
+
     @override_docstring(r_optional_extensible_flag)
     def p_optional_extensible_flag(self, p: P) -> None:
         extensible = len(p) == 2
@@ -517,6 +544,7 @@ def p_array_type(self, p: P) -> None:
             cap=p[3],
             extensible=p[5],
             token="{0}[{1}]".format(p[1], p[3]),
+            token_col_start=self._get_col(p, 1),
             lineno=p.lineno(2),
             filepath=self.current_filepath(),
         )
@@ -552,12 +580,15 @@ def p_open_enum_scope(self, p: P) -> None:
             name=p[2],
             type=p[4],
             token=p[2],
+            token_col_start=self._get_col(p, 2),
             lineno=p.lineno(2),
             filepath=self.current_filepath(),
             indent=self.current_indent(p),
             comment_block=self.collect_comment_block(),
             scope_stack=self.current_scope_stack(),
             _bound=self.current_proto(),
+            scope_start_lineno=p.lineno(5),  # '{'
+            scope_start_col=self._get_col(p, 5),  # '{'
         )
         self.push_scope(enum)
 
@@ -567,7 +598,10 @@ def p_enum_scope(self, p: P) -> None:
 
     @override_docstring(r_close_enum_scope)
     def p_close_enum_scope(self, p: P) -> None:
-        self.pop_scope().freeze()
+        enum = self.pop_scope()
+        enum.scope_end_lineno = p.lineno(1)
+        enum.scope_end_col = self._get_col(p, 1)
+        enum.freeze()
 
     @override_docstring(r_enum_items)
     def p_enum_items(self, p: P) -> None:
@@ -605,6 +639,7 @@ def p_enum_field(self, p: P) -> None:
             name=name,
             value=value,
             token=p[1],
+            token_col_start=self._get_col(p, 1),
             lineno=p.lineno(1),
             indent=self.current_indent(p),
             filepath=self.current_filepath(),
@@ -626,18 +661,24 @@ def p_open_message_scope(self, p: P) -> None:
             name=p[2],
             extensible=p[3],
             token=p[2],
+            token_col_start=self._get_col(p, 2),
             lineno=p.lineno(2),
             filepath=self.current_filepath(),
             indent=self.current_indent(p),
             comment_block=self.collect_comment_block(),
             scope_stack=self.current_scope_stack(),
             _bound=self.current_proto(),
+            scope_start_lineno=p.lineno(4),  # '{'
+            scope_start_col=self._get_col(p, 4),  # '{'
         )
         self.push_scope(message)
 
     @override_docstring(r_close_message_scope)
     def p_close_message_scope(self, p: P) -> None:
-        self.pop_scope().freeze()
+        message = self.pop_scope()
+        message.scope_end_lineno = p.lineno(1)  # '}'
+        message.scope_end_col = self._get_col(p, 1)  # '}'
+        message.freeze()
 
     @override_docstring(r_message_scope)
     def p_message_scope(self, p: P) -> None:
@@ -673,6 +714,7 @@ def p_message_field(self, p: P) -> None:
             type=type,
             number=field_number,
             token=p[2],
+            token_col_start=self._get_col(p, 2),
             lineno=p.lineno(2),
             filepath=self.current_filepath(),
             comment_block=self.collect_comment_block(),
@@ -685,6 +727,7 @@ def p_message_field(self, p: P) -> None:
     @override_docstring(r_message_field_name)
     def p_message_field_name(self, p: P) -> None:
         p[0] = p[1]
+        self.copy_p_tracking(p)  # from 1 to 0
 
     @override_docstring(r_boolean_literal)
     def p_boolean_literal(self, p: P) -> None:
@@ -700,7 +743,7 @@ def p_string_literal(self, p: P) -> None:
 
     @override_docstring(r_dotted_identifier)
     def p_dotted_identifier(self, p: P) -> None:
-        self.copy_p_tracking(p)
+        self.copy_p_tracking(p)  # from 1 => 0
         if len(p) == 4:
             p[0] = ".".join([p[1], p[3]])
         elif len(p) == 2:
@@ -716,6 +759,13 @@ def p_error(self, p: P) -> None:
             raise GrammarError(filepath=filepath, token=p.value(1), lineno=p.lineno(1))
         raise GrammarError()
 
+    def _get_col(self, p: P, k: int) -> int:
+        lexpos = p.lexpos(k)
+        # we dont use `last_newline_pos` here,
+        # because the recursive parsing may result a deeper `last_newline_pos`.
+        last_newline = p.lexer.lexdata.rfind("\n", 0, lexpos)
+        return lexpos - max(last_newline, 0)
+
 
 def parse(filepath: str, traditional_mode: bool = False) -> Proto:
     """Parse a bitproto from given filepath.
@@ -726,3 +776,14 @@ def parse(filepath: str, traditional_mode: bool = False) -> Proto:
        extensible grammar is used in traditional mode.
     """
     return Parser(traditional_mode=traditional_mode).parse(filepath)
+
+
+def parse_string(
+    content: str, traditional_mode: bool = False, filepath: str = ""
+) -> Proto:
+    """
+    Parse a bitproto from string.
+    """
+    return Parser(traditional_mode=traditional_mode).parse_string(
+        content, filepath=filepath
+    )

docs/language.rst

@@ -663,6 +663,13 @@ The bitproto naming guidelines are introduced in following code example:
 Editor Integration
 ^^^^^^^^^^^^^^^^^^
 
+Language Server
+""""""""""""""""
+
+A simple language server for bitproto: `bitproto-language-server github <https://github.com/hit9/bitproto/tree/master/editors/language_server>`_.
+
+This language_server is tested on neovim and vscode.
+
 Vim
 """
 A syntax plugin for `vim <https://www.vim.org/>`_ is available from