wip: start implementing multi language support

d-biehl · d-biehl · commit e6bc7a27c6bd · 2023-01-20T16:34:32.000+02:00
diff --git a/robotcode/language_server/common/decorators.py b/robotcode/language_server/common/decorators.py
@@ -1,21 +1,21 @@
-from typing import Any, Callable, List, Protocol, TypeVar, Union, runtime_checkable
+from typing import Any, Callable, List, Protocol, Set, TypeVar, Union, runtime_checkable
 
 from .text_document import TextDocument
 
 _F = TypeVar("_F", bound=Callable[..., Any])
 
 
-def language_id(id: str) -> Callable[[_F], _F]:
+def language_id(id: str, *ids: str) -> Callable[[_F], _F]:
     def decorator(func: _F) -> _F:
-        setattr(func, "__language_id__", id)
+        setattr(func, "__language_id__", {id, *ids})
         return func
 
     return decorator
 
 
 @runtime_checkable
 class HasLanguageId(Protocol):
-    __language_id__: str
+    __language_id__: Set[str]
 
 
 def trigger_characters(characters: List[str]) -> Callable[[_F], _F]:
@@ -72,10 +72,13 @@ class HasCodeActionKinds(Protocol):
 
 def language_id_filter(language_id_or_document: Union[str, TextDocument]) -> Callable[[Any], bool]:
     def filter(c: Any) -> bool:
-        return not isinstance(c, HasLanguageId) or c.__language_id__ == (
-            language_id_or_document.language_id
-            if isinstance(language_id_or_document, TextDocument)
-            else language_id_or_document
+        return not isinstance(c, HasLanguageId) or (
+            (
+                language_id_or_document.language_id
+                if isinstance(language_id_or_document, TextDocument)
+                else language_id_or_document
+            )
+            in c.__language_id__
         )
 
     return filter
diff --git a/robotcode/language_server/common/parts/documents.py b/robotcode/language_server/common/parts/documents.py
@@ -3,6 +3,7 @@
 import asyncio
 import os
 import re
+from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -119,7 +120,7 @@ async def _file_watcher(self, sender: Any, changes: List[FileEvent]) -> None:
                         loop=self.parent.loop,
                     )
 
-    async def read_document_text(self, uri: Uri, language_id: Union[str, Callable[[Any], bool]]) -> str:
+    async def read_document_text(self, uri: Uri, language_id: Union[str, Callable[[Any], bool], None]) -> str:
         for e in await self.on_read_document_text(
             self, uri, callback_filter=language_id_filter(language_id) if isinstance(language_id, str) else language_id
         ):
@@ -128,9 +129,18 @@ async def read_document_text(self, uri: Uri, language_id: Union[str, Callable[[A
 
         raise FileNotFoundError(str(uri))
 
+    def detect_language_id(self, path_or_uri: Union[str, os.PathLike[Any], Uri]) -> str:
+        path = path_or_uri.to_path() if isinstance(path_or_uri, Uri) else Path(path_or_uri)
+
+        for lang in self.parent.languages:
+            if path.suffix in lang.extensions:
+                return lang.id
+
+        return "unknown"
+
     @_logger.call
     async def get_or_open_document(
-        self, path: Union[str, os.PathLike[Any]], language_id: str, version: Optional[int] = None
+        self, path: Union[str, os.PathLike[Any]], language_id: Optional[str] = None, version: Optional[int] = None
     ) -> TextDocument:
         uri = Uri.from_path(path).normalized()
 
@@ -141,7 +151,7 @@ async def get_or_open_document(
         try:
             return await self.parent.documents.append_document(
                 document_uri=DocumentUri(uri),
-                language_id=language_id,
+                language_id=language_id or self.detect_language_id(path),
                 text=await self.read_document_text(uri, language_id),
                 version=version,
             )
diff --git a/robotcode/language_server/common/parts/inline_value.py b/robotcode/language_server/common/parts/inline_value.py
@@ -46,7 +46,8 @@ def extend_capabilities(self, capabilities: ServerCapabilities) -> None:
             document_filters: DocumentSelector = []
             for e in self.collect:
                 if isinstance(e, HasLanguageId):
-                    document_filters.append(DocumentFilter(language=e.__language_id__))
+                    for lang_id in e.__language_id__:
+                        document_filters.append(DocumentFilter(language=lang_id))
             capabilities.inline_value_provider = InlineValueRegistrationOptions(
                 work_done_progress=True, document_selector=document_filters if document_filters else None
             )
diff --git a/robotcode/language_server/common/parts/workspace.py b/robotcode/language_server/common/parts/workspace.py
@@ -308,21 +308,23 @@ async def _workspace_will_delete_files(self, files: List[FileDelete], *args: Any
     async def _workspace_did_delete_files(self, files: List[FileDelete], *args: Any, **kwargs: Any) -> None:
         await self.did_delete_files(self, list(f.uri for f in files))
 
-    async def get_configuration(self, section: Type[_TConfig], scope_uri: Union[str, Uri, None] = None) -> _TConfig:
+    async def get_configuration(
+        self, section: Type[_TConfig], scope_uri: Union[str, Uri, None] = None, request: bool = True
+    ) -> _TConfig:
         config = await self.get_configuration_raw(
-            section=cast(HasConfigSection, section).__config_section__, scope_uri=scope_uri
+            section=cast(HasConfigSection, section).__config_section__, scope_uri=scope_uri, request=request
         )
 
         return from_dict(config if config is not None else {}, section)
 
     async def get_configuration_raw(
-        self, section: Optional[str], scope_uri: Union[str, Uri, None] = None
+        self, section: Optional[str], scope_uri: Union[str, Uri, None] = None, request: bool = True
     ) -> Optional[Any]:
-
         if (
             self.parent.client_capabilities
             and self.parent.client_capabilities.workspace
             and self.parent.client_capabilities.workspace.configuration
+            and request
         ):
             r = await self.parent.send_request_async(
                 "workspace/configuration",
diff --git a/robotcode/language_server/robotframework/configuration.py b/robotcode/language_server/robotframework/configuration.py
@@ -41,6 +41,7 @@ class RobotConfig(ConfigBase):
     log_level: Optional[str] = None
     mode: Optional[RpaMode] = None
     languages: Optional[List[str]] = None
+    parsers: Optional[List[str]] = None
 
     def get_rpa_mode(self) -> Optional[bool]:
         if self.mode == RpaMode.RPA:
diff --git a/robotcode/language_server/robotframework/diagnostics/imports_manager.py b/robotcode/language_server/robotframework/diagnostics/imports_manager.py
@@ -500,7 +500,6 @@ def __init__(self, parent_protocol: RobotLanguageServerProtocol, folder: Uri, co
         self.parent_protocol = parent_protocol
 
         self.folder = folder
-        get_robot_version()
 
         cache_base_path = self.folder.to_path()
         if (
@@ -1339,7 +1338,7 @@ async def _get_document() -> TextDocument:
                     f"Supported extensions are {', '.join(repr(s) for s in RESOURCE_EXTENSIONS)}."
                 )
 
-            return await self.parent_protocol.documents.get_or_open_document(source_path, "robotframework")
+            return await self.parent_protocol.documents.get_or_open_document(source_path)
 
         entry_key = _ResourcesEntryKey(source)
 
diff --git a/robotcode/language_server/robotframework/parts/discovering.py b/robotcode/language_server/robotframework/parts/discovering.py
@@ -251,6 +251,7 @@ async def generate(suite: TestSuite) -> TestItem:
             config = await self.get_config(workspace_folder)
             rpa_mode = config.get_rpa_mode() if config is not None else None
             languages = await self.parent.documents_cache.get_workspace_languages(workspace_folder)
+            parsers = config.parsers if config is not None else None
 
             if paths is None and config is not None:
                 paths = config.paths
@@ -283,7 +284,14 @@ def nonexisting_paths(paths: List[str]) -> Iterator[str]:
 
                 valid_paths = [i for i in normalize_paths(paths)]
 
-                if get_robot_version() >= (6, 0):
+                if get_robot_version() >= (6, 1):
+                    builder = TestSuiteBuilder(
+                        included_suites=suites if suites else None,
+                        rpa=rpa_mode,
+                        lang=languages,
+                        parsers=parsers,
+                    )
+                elif get_robot_version() >= (6, 0):
                     builder = TestSuiteBuilder(
                         included_suites=suites if suites else None,
                         rpa=rpa_mode,
@@ -318,12 +326,15 @@ def nonexisting_paths(paths: List[str]) -> Iterator[str]:
                     )
                 ]
             else:
-                if get_robot_version() >= (6, 0):
+                if get_robot_version() >= (6, 1):
                     builder = TestSuiteBuilder(
                         included_suites=suites if suites else None,
                         rpa=rpa_mode,
                         lang=languages,
+                        parsers=parsers,
                     )
+                elif get_robot_version() >= (6, 0):
+                    builder = TestSuiteBuilder(included_suites=suites if suites else None, rpa=rpa_mode, lang=languages)
                 else:
                     builder = TestSuiteBuilder(included_suites=suites if suites else None, rpa=rpa_mode)
                 return [await generate(builder.build(str(workspace_path)))]
diff --git a/robotcode/language_server/robotframework/parts/documents_cache.py b/robotcode/language_server/robotframework/parts/documents_cache.py
@@ -131,7 +131,7 @@ async def __get_document_type(self, document: TextDocument) -> DocumentType:
         elif suffix == ".resource":
             return DocumentType.RESOURCE
         else:
-            return DocumentType.GENERAL
+            return DocumentType.UNKNOWN
 
     async def get_tokens(self, document: TextDocument, data_only: bool = False) -> List[Token]:
         if data_only:
diff --git a/robotcode/language_server/robotframework/parts/hover.py b/robotcode/language_server/robotframework/parts/hover.py
@@ -66,7 +66,11 @@ def _find_method(self, cls: Type[Any]) -> Optional[_HoverMethod]:
     @_logger.call
     async def collect(self, sender: Any, document: TextDocument, position: Position) -> Optional[Hover]:
 
-        result_nodes = await get_nodes_at_position(await self.parent.documents_cache.get_model(document), position)
+        model = await self.parent.documents_cache.get_model(document)
+        if model is None:
+            return None
+
+        result_nodes = await get_nodes_at_position(model, position)
 
         if not result_nodes:
             return None
diff --git a/robotcode/language_server/robotframework/parts/references.py b/robotcode/language_server/robotframework/parts/references.py
@@ -148,12 +148,12 @@ async def _find_references_in_workspace(
 
         # tasks = []
         for doc in self.parent.documents.documents:
-            if doc.language_id == "robotframework":
-                result.extend(await func(doc, *args, **kwargs))
-                if result and stop_at_first:
-                    break
+            # if doc.language_id == "robotframework":
+            result.extend(await func(doc, *args, **kwargs))
+            if result and stop_at_first:
+                break
 
-                # tasks.append(run_coroutine_in_thread(func, doc, *args, **kwargs))
+            # tasks.append(run_coroutine_in_thread(func, doc, *args, **kwargs))
 
         # result = await asyncio.gather(*tasks)
 
diff --git a/robotcode/language_server/robotframework/parts/robot_workspace.py b/robotcode/language_server/robotframework/parts/robot_workspace.py
@@ -53,7 +53,7 @@ async def on_file_changed(self, sender: Any, files: List[FileEvent]) -> None:  #
             try:
                 path = doc_uri.to_path()
                 if path.suffix in [ROBOT_FILE_EXTENSION, RESOURCE_FILE_EXTENSION]:
-                    document = await self.parent.documents.get_or_open_document(path, "robotframework")
+                    document = await self.parent.documents.get_or_open_document(path)
                     if not document.opened_in_editor:
                         await (await self.parent.documents_cache.get_namespace(document)).ensure_initialized()
 
@@ -103,7 +103,7 @@ async def _load_workspace_documents(self, sender: Any) -> List[WorkspaceDocument
                     try:
                         for i, f in enumerate(files):
                             try:
-                                await self.parent.documents.get_or_open_document(f, "robotframework")
+                                await self.parent.documents.get_or_open_document(f)
 
                                 if config.analysis.progress_mode != AnalysisProgressMode.OFF:
                                     name = f.relative_to(folder.uri.to_path())
diff --git a/robotcode/language_server/robotframework/protocol.py b/robotcode/language_server/robotframework/protocol.py
@@ -2,7 +2,7 @@
 import sys
 from dataclasses import dataclass
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from ...__version__ import __version__
 from ...jsonrpc2.protocol import (
@@ -14,7 +14,7 @@
 from ...utils.logging import LoggingDescriptor
 from ..common.lsp_types import InitializeError, Model
 from ..common.parts.document_symbols import symbol_information_label
-from ..common.protocol import LanguageServerProtocol
+from ..common.protocol import LanguageDefinition, LanguageServerProtocol
 from .configuration import RobotConfig
 from .parts.code_action_documentation import RobotCodeActionDocumentationProtocolPart
 from .parts.code_action_fixes import RobotCodeActionFixesProtocolPart
@@ -108,7 +108,21 @@ class RobotLanguageServerProtocol(LanguageServerProtocol):
     short_name = "RobotCode"
     version = __version__
 
-    file_extensions = {"robot", "resource", "py", "yaml", "yml"}
+    file_extensions = {"robot", "resource", "py", "yaml", "yml", "feature"}
+
+    languages: List[LanguageDefinition] = [
+        LanguageDefinition(
+            id="robotframework",
+            extensions=[".robot", ".resource"],
+            aliases=["Robot Framework", "robotframework"],
+        ),
+        # LanguageDefinition(
+        #     id="feature",
+        #     extensions=[".feature", ".md"],
+        #     aliases=["feature", "gherkin", "Gherkin", "cucumber"],
+        # ),
+        LanguageDefinition(id="markdown", extensions=[".md"]),
+    ]
 
     def __init__(self, server: "RobotLanguageServer"):
         super().__init__(server)
@@ -135,14 +149,14 @@ async def _on_initialize(self, sender: Any, initialization_options: Optional[Any
 
         self.workspace.did_change_configuration.add(self._on_did_change_configuration)
 
-        self._logger.debug(f"initialized with {repr(self.options)}")
+        self._logger.critical(f"initialized with {repr(self.options)}")
 
     async def _on_did_change_configuration(self, sender: Any, settings: Dict[str, Any]) -> None:
         pass
 
     async def _on_initialized(self, sender: Any) -> None:
         for folder in self.workspace.workspace_folders:
-            config: RobotConfig = await self.workspace.get_configuration(RobotConfig, folder.uri)
+            config: RobotConfig = await self.workspace.get_configuration(RobotConfig, folder.uri, request=False)
             if config is not None:
                 if config.env:
                     for k, v in config.env.items():
diff --git a/vscode-client/debugmanager.ts b/vscode-client/debugmanager.ts
@@ -5,7 +5,7 @@ import * as path from "path";
 import * as vscode from "vscode";
 import { PythonManager } from "./pythonmanger";
 import { CONFIG_SECTION } from "./config";
-import { LanguageClientsManager, toVsCodeRange } from "./languageclientsmanger";
+import { LanguageClientsManager, SUPPORTED_LANGUAGES, toVsCodeRange } from "./languageclientsmanger";
 import { WeakValueSet } from "./utils";
 
 const DEBUG_ADAPTER_DEFAULT_TCP_PORT = 6611;
@@ -55,7 +55,11 @@ class RobotCodeDebugConfigurationProvider implements vscode.DebugConfigurationPr
   ): Promise<vscode.DebugConfiguration> {
     if (!debugConfiguration.type && !debugConfiguration.request && !debugConfiguration.name) {
       const editor = vscode.window.activeTextEditor;
-      if (editor && editor.document.languageId === "robotframework" && editor.document.fileName.endsWith(".robot")) {
+      if (
+        editor &&
+        SUPPORTED_LANGUAGES.includes(editor.document.languageId) &&
+        (editor.document.fileName.endsWith(".robot") || editor.document.fileName.endsWith(".feature"))
+      ) {
         const result = await vscode.window.showQuickPick(
           DEBUG_CONFIGURATIONS.map((v) => v),
           { canPickMany: false },
diff --git a/vscode-client/languageclientsmanger.ts b/vscode-client/languageclientsmanger.ts
@@ -35,6 +35,8 @@ export function toVsCodeRange(range: Range): vscode.Range {
   );
 }
 
+export const SUPPORTED_LANGUAGES = ["robotframework"];
+
 export interface RobotTestItem {
   type: string;
   id: string;
@@ -299,7 +301,7 @@ export class LanguageClientsManager {
   }
 
   public async getLanguageClientForDocument(document: vscode.TextDocument): Promise<LanguageClient | undefined> {
-    if (document.languageId !== "robotframework") return;
+    if (!SUPPORTED_LANGUAGES.includes(document.languageId)) return;
 
     return this.getLanguageClientForResource(document.uri);
   }
@@ -348,9 +350,16 @@ export class LanguageClientsManager {
 
       const clientOptions: LanguageClientOptions = {
         documentSelector:
+          // TODO: use SUPPORTED_LANGUAGES here
           vscode.workspace.workspaceFolders?.length === 1
-            ? [{ scheme: "file", language: "robotframework" }]
-            : [{ scheme: "file", language: "robotframework", pattern: `${workspaceFolder.uri.fsPath}/**/*` }],
+            ? [
+                { scheme: "file", language: "robotframework" },
+                { scheme: "file", language: "feature" },
+              ]
+            : [
+                { scheme: "file", language: "robotframework", pattern: `${workspaceFolder.uri.fsPath}/**/*` },
+                { scheme: "file", language: "feature", pattern: `${workspaceFolder.uri.fsPath}/**/*` },
+              ],
         synchronize: {
           configurationSection: [CONFIG_SECTION],
         },
@@ -532,7 +541,7 @@ export class LanguageClientsManager {
     }
 
     for (const document of vscode.workspace.textDocuments) {
-      if (document.languageId === "robotframework") {
+      if (SUPPORTED_LANGUAGES.includes(document.languageId)) {
         const workspaceFolder = vscode.workspace.getWorkspaceFolder(document.uri);
         if (workspaceFolder) {
           folders.add(workspaceFolder);
diff --git a/vscode-client/testcontrollermanager.ts b/vscode-client/testcontrollermanager.ts
