some refactoring to speedup loading and parsing documents

Author: d-biehl

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to the "robotcode" extension will be documented in this file
 
 ## [Unreleased]
 
+### added
+
+- some refactoring to speedup loading and parsing documents
+- semantic tokens now highlight builtin keywords
+
 ##  0.3.2
 
 ### added

robotcode/language_server/common/parts/documents.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import gc
+import re
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Mapping, Optional
 
 from ....jsonrpc2.protocol import JsonRPCException, rpc_method
@@ -103,45 +104,55 @@ def __hash__(self) -> int:
 
     def _create_document(
         self,
-        text_document_item: Optional[TextDocumentItem] = None,
-        *,
-        document_uri: Optional[DocumentUri] = None,
+        document_uri: DocumentUri,
+        text: str,
         language_id: Optional[str] = None,
         version: Optional[int] = None,
-        text: Optional[str] = None,
     ) -> TextDocument:
         return TextDocument(
-            text_document_item=text_document_item,
             document_uri=document_uri,
             language_id=language_id,
-            version=version,
             text=text,
+            version=version,
         )
 
     def append_document(
         self,
         document_uri: DocumentUri,
         language_id: str,
-        text: Optional[str],
+        text: str,
         version: Optional[int] = None,
     ) -> TextDocument:
-        document = self._create_document(document_uri=document_uri, language_id=language_id, version=version, text=text)
+        document = self._create_document(document_uri=document_uri, language_id=language_id, text=text, version=version)
 
         self._documents[document_uri] = document
         return document
 
+    __NORMALIZE_LINE_ENDINGS = re.compile(r"(\r?\n)")
+
+    @classmethod
+    def _normalize_line_endings(cls, text: str) -> str:
+        return cls.__NORMALIZE_LINE_ENDINGS.sub("\n", text)
+
     @rpc_method(name="textDocument/didOpen", param_type=DidOpenTextDocumentParams)
     @_logger.call
     async def _text_document_did_open(self, text_document: TextDocumentItem, *args: Any, **kwargs: Any) -> None:
         uri = str(Uri(text_document.uri).normalized())
         document = self.get(uri, None)
 
+        text_changed = True
+        normalized_text = self._normalize_line_endings(text_document.text)
+
         if document is None:
-            document = self._create_document(text_document)
+            document = self._create_document(
+                text_document.uri, normalized_text, text_document.language_id, text_document.version
+            )
 
             self._documents[uri] = document
         else:
-            await document.apply_full_change(text_document.version, text_document.text)
+            text_changed = document.text != normalized_text
+            if text_changed:
+                await document.apply_full_change(text_document.version, normalized_text)
 
         document.opened_in_editor = True
         document.references.add(self)
@@ -152,6 +163,13 @@ async def _text_document_did_open(self, text_document: TextDocumentItem, *args:
             callback_filter=lambda c: not isinstance(c, HasLanguageId) or c.__language_id__ == document.language_id,
         )
 
+        if text_changed:
+            await self.did_change(
+                self,
+                document,
+                callback_filter=lambda c: not isinstance(c, HasLanguageId) or c.__language_id__ == document.language_id,
+            )
+
     @rpc_method(name="textDocument/didClose", param_type=DidCloseTextDocumentParams)
     @_logger.call
     async def _text_document_did_close(self, text_document: TextDocumentIdentifier, *args: Any, **kwargs: Any) -> None:
@@ -195,8 +213,20 @@ async def _text_document_did_save(
         document = self._documents.get(str(Uri(text_document.uri).normalized()), None)
         self._logger.warning(lambda: f"Document {text_document.uri} is not opened.", condition=lambda: document is None)
 
-        if document is not None and text is not None:
-            await document.apply_full_change(None, text)
+        if document is not None:
+            if text is not None:
+                normalized_text = self._normalize_line_endings(text)
+
+                text_changed = document.text != normalized_text
+                if text_changed:
+                    await document.apply_full_change(None, text)
+
+                    await self.did_change(
+                        self,
+                        document,
+                        callback_filter=lambda c: not isinstance(c, HasLanguageId)
+                        or c.__language_id__ == document.language_id,
+                    )
 
             await self.did_save(
                 self,
@@ -238,12 +268,14 @@ async def _text_document_did_change(
             elif sync_kind == TextDocumentSyncKind.FULL and isinstance(
                 content_change, TextDocumentContentTextChangeEvent
             ):
-                await document.apply_full_change(text_document.version, content_change.text)
+                await document.apply_full_change(
+                    text_document.version, self._normalize_line_endings(content_change.text)
+                )
             elif sync_kind == TextDocumentSyncKind.INCREMENTAL and isinstance(
                 content_change, TextDocumentContentRangeChangeEvent
             ):
                 await document.apply_incremental_change(
-                    text_document.version, content_change.range, content_change.text
+                    text_document.version, content_change.range, self._normalize_line_endings(content_change.text)
                 )
             else:
                 raise LanguageServerDocumentException(

robotcode/language_server/common/text_document.py

@@ -8,7 +8,7 @@
 from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union, cast
 
 from ...utils.uri import Uri
-from .lsp_types import DocumentUri, Position, Range, TextDocumentItem
+from .lsp_types import DocumentUri, Position, Range
 
 
 def _utf16_unit_offset(chars: str) -> int:
@@ -47,37 +47,25 @@ def __init__(self, data: Any = None) -> None:
 class TextDocument:
     def __init__(
         self,
-        text_document_item: Optional[TextDocumentItem] = None,
-        *,
-        document_uri: Optional[DocumentUri] = None,
+        document_uri: DocumentUri,
+        text: str,
         language_id: Optional[str] = None,
         version: Optional[int] = None,
-        text: Optional[str] = None,
     ) -> None:
         super().__init__()
 
         self._lock = asyncio.Lock()
 
         self._references: weakref.WeakSet[Any] = weakref.WeakSet()
 
-        self.document_uri = (
-            text_document_item.uri
-            if text_document_item is not None
-            else document_uri
-            if document_uri is not None
-            else ""
-        )
+        self.document_uri = document_uri
+
         self.uri = Uri(self.document_uri).normalized()
 
-        self.language_id = (
-            text_document_item.language_id
-            if text_document_item is not None
-            else language_id
-            if language_id is not None
-            else ""
-        )
-        self.version = text_document_item.version if text_document_item is not None else version
-        self._text = text_document_item.text if text_document_item is not None else text if text is not None else ""
+        self.language_id = language_id
+
+        self.version = version
+        self._text = text
 
         self._lines: Optional[List[str]] = None
 

robotcode/language_server/robotframework/diagnostics/imports_manager.py

@@ -380,10 +380,7 @@ def __init__(
         self._resources: OrderedDict[_ResourcesEntryKey, _ResourcesEntry] = OrderedDict()
         self.file_watchers: List[FileWatcherEntry] = []
         self._loop = asyncio.get_event_loop()
-        self.parent_protocol.documents.did_open.add(self.resource_document_changed)
         self.parent_protocol.documents.did_change.add(self.resource_document_changed)
-        self.parent_protocol.documents.did_close.add(self.resource_document_changed)
-        self.parent_protocol.documents.did_save.add(self.resource_document_changed)
 
     @async_tasking_event
     async def libraries_changed(sender, libraries: List[LibraryDoc]) -> None:

robotcode/language_server/robotframework/diagnostics/library_doc.py

@@ -274,13 +274,16 @@ class KeywordDoc(Model):
     is_error_handler: bool = False
     error_handler_message: Optional[str] = None
     is_initializer: bool = False
+    is_registered_run_keyword: bool = False
+    args_to_process: Optional[int] = None
+    deprecated: bool = False
 
     def __str__(self) -> str:
         return f"{self.name}({', '.join(str(arg) for arg in self.args)})"
 
     @property
     def is_deprecated(self) -> bool:
-        return DEPRECATED_PATTERN.match(self.doc) is not None
+        return self.deprecated or DEPRECATED_PATTERN.match(self.doc) is not None
 
     @property
     def deprecated_message(self) -> str:
@@ -946,6 +949,7 @@ def get_library_doc(
     from robot.output import LOGGER
     from robot.output.loggerhelper import AbstractLogger
     from robot.running.outputcapture import OutputCapturer
+    from robot.running.runkwregister import RUN_KW_REGISTER
     from robot.running.testlibraries import _get_lib_class
     from robot.utils import Importer
 
@@ -1128,6 +1132,9 @@ def get_test_library(
                             doc_format=str(lib.doc_format) or DEFAULT_DOC_FORMAT,
                             is_error_handler=kw[1].is_error_handler,
                             error_handler_message=kw[1].error_handler_message,
+                            is_registered_run_keyword=RUN_KW_REGISTER.is_run_keyword(libdoc.name, kw[0].name),
+                            args_to_process=RUN_KW_REGISTER.get_args_to_process(libdoc.name, kw[0].name),
+                            deprecated=kw[0].deprecated,
                         )
                         for kw in [
                             (KeywordDocBuilder().build_keyword(k), k)

robotcode/language_server/robotframework/diagnostics/namespace.py

@@ -922,12 +922,13 @@ def __init__(
         self._imports: Optional[List[Import]] = None
         self._own_variables: Optional[List[VariableDefinition]] = None
         self._diagnostics: List[Diagnostic] = []
-
         self._keywords: Optional[List[KeywordDoc]] = None
 
         # TODO: how to get the search order from model
         self.search_order: Tuple[str, ...] = ()
 
+        self._finder: Optional[KeywordFinder] = None
+
     @property
     def document(self) -> Optional[TextDocument]:
         return self._document() if self._document is not None else None
@@ -938,7 +939,7 @@ async def libraries_changed(self, sender: Any, libraries: List[LibraryDoc]) -> N
             if any(e for e in self._libraries.values() if e.library_doc == p):
                 if self.document is not None:
                     self.document.set_data(Namespace.DataEntry, None)
-                self.invalidated_callback(self)
+                await self.invalidate()
                 break
 
     @_logger.call
@@ -947,9 +948,27 @@ async def resources_changed(self, sender: Any, resources: List[LibraryDoc]) -> N
             if any(e for e in self._resources.values() if e.library_doc.source == p.source):
                 if self.document is not None:
                     self.document.set_data(Namespace.DataEntry, None)
-                self.invalidated_callback(self)
+                await self.invalidate()
                 break
 
+    async def invalidate(self) -> None:
+        async with self._initialize_lock, self._library_doc_lock, self._analyze_lock:
+            self._initialized = False
+
+            self._libraries = OrderedDict()
+            self._libraries_matchers = None
+            self._resources = OrderedDict()
+            self._resources_matchers = None
+            self._variables = OrderedDict()
+            self._imports = None
+            self._own_variables = None
+            self._keywords = None
+            self._library_doc = None
+            self._analyzed = False
+            self._diagnostics = []
+
+        self.invalidated_callback(self)
+
     @_logger.call
     async def get_diagnostisc(self) -> List[Diagnostic]:
         await self.ensure_initialized()
@@ -1338,7 +1357,7 @@ async def _import(value: Import) -> Optional[LibraryEntry]:
                                 )
 
                 else:
-                    if entry.name == BUILTIN_LIBRARY_NAME and entry.alias is None:
+                    if top_level and entry.name == BUILTIN_LIBRARY_NAME and entry.alias is None:
                         self._diagnostics.append(
                             Diagnostic(
                                 range=entry.import_range,
@@ -1502,9 +1521,18 @@ async def _analyze(self) -> None:
 
     @_logger.call
     async def find_keyword(self, name: Optional[str]) -> Optional[KeywordDoc]:
+        if self._finder is None:
+            await self.ensure_initialized()
+
+            self._finder = await self.create_finder()
+
+        return await self._finder.find_keyword(name)
+
+    @_logger.call
+    async def create_finder(self) -> KeywordFinder:
         await self.ensure_initialized()
 
-        return await KeywordFinder(self).find_keyword(name)
+        return KeywordFinder(self)
 
 
 class DiagnosticsEntry(NamedTuple):
@@ -1523,6 +1551,7 @@ def __init__(self, namespace: Namespace) -> None:
         self.namespace = namespace
         self.diagnostics: List[DiagnosticsEntry] = []
         self.self_library_doc: Optional[LibraryDoc] = None
+        self._cache: Dict[Optional[str], Tuple[Optional[KeywordDoc], List[DiagnosticsEntry]]] = {}
 
     def reset_diagnostics(self) -> None:
         self.diagnostics = []
@@ -1531,15 +1560,22 @@ async def find_keyword(self, name: Optional[str]) -> Optional[KeywordDoc]:
         try:
             self.reset_diagnostics()
 
-            result = await self._find_keyword(name)
-            if result is None:
-                self.diagnostics.append(
-                    DiagnosticsEntry(
-                        f"No keyword with name {repr(name)} found.", DiagnosticSeverity.ERROR, "KeywordError"
+            cached = self._cache.get(name, None)
+
+            if cached is not None:
+                self.diagnostics = cached[1]
+                return cached[0]
+            else:
+                result = await self._find_keyword(name)
+                if result is None:
+                    self.diagnostics.append(
+                        DiagnosticsEntry(
+                            f"No keyword with name {repr(name)} found.", DiagnosticSeverity.ERROR, "KeywordError"
+                        )
                     )
-                )
+                self._cache[name] = (result, self.diagnostics)
 
-            return result
+                return result
         except CancelSearchError:
             return None