Skip to content

Commit 9554592

Browse files
TechPizzaDevTechPizzaDev
committed
Initial attempt at LSP based on python-lsp/python-lsp-server#533 
1 parent d5bb10f commit 9554592

File tree

6 files changed

+288
-1
lines changed

6 files changed

+288
-1
lines changed

‎jedi_language_server/constants.py

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,17 @@
11
"""Constants."""
22

3+
from lsprotocol.types import SemanticTokenTypes
4+
35
MAX_CONCURRENT_DEBOUNCE_CALLS = 10
46
"""The maximum number of concurrent calls allowed by the debounce decorator."""
7+
8+
_token_types = list(SemanticTokenTypes)
9+
10+
TYPE_TO_TOKEN_ID = {
11+
"module": _token_types.index(SemanticTokenTypes.Namespace),
12+
"class": _token_types.index(SemanticTokenTypes.Class),
13+
"function": _token_types.index(SemanticTokenTypes.Function),
14+
"param": _token_types.index(SemanticTokenTypes.Parameter),
15+
"statement": _token_types.index(SemanticTokenTypes.Variable),
16+
"property": _token_types.index(SemanticTokenTypes.Property),
17+
}

‎jedi_language_server/server.py

Lines changed: 155 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
from typing import Any, List, Optional, Union
1111

1212
import cattrs
13-
from jedi import Project, __version__
13+
from jedi import Project, Script, __version__
14+
from jedi.api.classes import Name
1415
from jedi.api.refactoring import RefactoringError
1516
from lsprotocol.types import (
1617
COMPLETION_ITEM_RESOLVE,
@@ -32,6 +33,8 @@
3233
TEXT_DOCUMENT_HOVER,
3334
TEXT_DOCUMENT_REFERENCES,
3435
TEXT_DOCUMENT_RENAME,
36+
TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
37+
TEXT_DOCUMENT_SEMANTIC_TOKENS_RANGE,
3538
TEXT_DOCUMENT_SIGNATURE_HELP,
3639
TEXT_DOCUMENT_TYPE_DEFINITION,
3740
WORKSPACE_DID_CHANGE_CONFIGURATION,
@@ -67,7 +70,14 @@
6770
NotebookDocumentSyncOptionsNotebookSelectorType2,
6871
NotebookDocumentSyncOptionsNotebookSelectorType2CellsType,
6972
ParameterInformation,
73+
Position,
74+
Range,
7075
RenameParams,
76+
SemanticTokens,
77+
SemanticTokensLegend,
78+
SemanticTokensParams,
79+
SemanticTokensRangeParams,
80+
SemanticTokenTypes,
7181
SignatureHelp,
7282
SignatureHelpOptions,
7383
SignatureInformation,
@@ -76,11 +86,13 @@
7686
WorkspaceEdit,
7787
WorkspaceSymbolParams,
7888
)
89+
from lsprotocol.validators import INTEGER_MAX_VALUE
7990
from pygls.capabilities import get_capability
8091
from pygls.protocol import LanguageServerProtocol, lsp_method
8192
from pygls.server import LanguageServer
8293

8394
from . import jedi_utils, notebook_utils, pygls_utils, text_edit_utils
95+
from .constants import TYPE_TO_TOKEN_ID
8496
from .initialization_options import (
8597
InitializationOptions,
8698
initialization_options_converter,
@@ -724,6 +736,148 @@ def did_change_configuration(
724736
"""
725737

726738

739+
def _raw_semantic_token(
740+
server: JediLanguageServer, n: Name
741+
) -> Union[list[int], None]:
742+
"""Find an appropriate semantic token for the name.
743+
744+
This works by looking up the definition (using jedi ``goto``) of the name and
745+
matching the definition's type to one of the availabile semantic tokens. Further
746+
improvements are possible by inspecting context, e.g. semantic token modifiers such
747+
as ``abstract`` or ``async`` or even different tokens, e.g. ``property`` or
748+
``method``. Dunder methods may warrant special treatment/modifiers as well.
749+
The return is a "raw" semantic token rather than a "diff." This is in the form of a
750+
length 5 array of integers where the elements are the line number, starting
751+
character, length, token index, and modifiers (as an integer whose binary
752+
representation has bits set at the indices of all applicable modifiers).
753+
"""
754+
definitions: list[Name] = n.goto(
755+
follow_imports=True,
756+
follow_builtin_imports=True,
757+
only_stubs=False,
758+
prefer_stubs=False,
759+
)
760+
if not definitions:
761+
server.show_message_log(
762+
f"no definitions found for name {n.description} ({n.line}:{n.column})",
763+
MessageType.Debug,
764+
)
765+
return None
766+
767+
if len(definitions) > 1:
768+
server.show_message_log(
769+
f"multiple definitions found for name {n.description} ({n.line}:{n.column})",
770+
MessageType.Debug,
771+
)
772+
773+
definition, *_ = definitions
774+
if (
775+
definition_type := TYPE_TO_TOKEN_ID.get(definition.type, None)
776+
) is None:
777+
server.show_message_log(
778+
f"no matching semantic token for name {n.description} ({n.line}:{n.column})",
779+
MessageType.Debug,
780+
)
781+
return None
782+
783+
return [n.line - 1, n.column, len(n.name), definition_type, 0]
784+
785+
786+
@SERVER.thread()
787+
@SERVER.feature(
788+
TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
789+
SemanticTokensLegend(
790+
token_types=list(SemanticTokenTypes), token_modifiers=[]
791+
),
792+
)
793+
def semantic_tokens_full(
794+
server: JediLanguageServer, params: SemanticTokensParams
795+
) -> SemanticTokens:
796+
"""Thin wrap around _semantic_tokens_range()."""
797+
document = server.workspace.get_text_document(params.text_document.uri)
798+
jedi_script = jedi_utils.script(server.project, document)
799+
800+
server.show_message_log(
801+
f"semantic_tokens_full {params.text_document.uri} ",
802+
MessageType.Log,
803+
)
804+
805+
return _semantic_tokens_range(
806+
server,
807+
jedi_script,
808+
Range(Position(0, 0), Position(INTEGER_MAX_VALUE, INTEGER_MAX_VALUE)),
809+
)
810+
811+
812+
@SERVER.thread()
813+
@SERVER.feature(
814+
TEXT_DOCUMENT_SEMANTIC_TOKENS_RANGE,
815+
SemanticTokensLegend(
816+
token_types=list(SemanticTokenTypes), token_modifiers=[]
817+
),
818+
)
819+
def semantic_tokens_range(
820+
server: JediLanguageServer, params: SemanticTokensRangeParams
821+
) -> SemanticTokens:
822+
"""Thin wrap around _semantic_tokens_range()."""
823+
document = server.workspace.get_text_document(params.text_document.uri)
824+
jedi_script = jedi_utils.script(server.project, document)
825+
826+
server.show_message_log(
827+
f"semantic_tokens_range {params.text_document.uri} {params.range}",
828+
MessageType.Log,
829+
)
830+
831+
return _semantic_tokens_range(server, jedi_script, params.range)
832+
833+
834+
def _semantic_tokens_range(
835+
server: JediLanguageServer, jedi_script: Script, doc_range: Range
836+
) -> SemanticTokens:
837+
"""General purpose function to do full / range semantic tokens."""
838+
line, column = doc_range.start.line, doc_range.start.character
839+
names = jedi_script.get_names(
840+
all_scopes=True, definitions=True, references=True
841+
)
842+
data = []
843+
844+
for n in names:
845+
if (
846+
not doc_range.start
847+
< Position(n.line - 1, n.column)
848+
< doc_range.end
849+
):
850+
continue
851+
852+
token = _raw_semantic_token(server, n)
853+
854+
server.show_message_log(
855+
f"raw token for name {n.description} ({n.line - 1}:{n.column}): {token}",
856+
MessageType.Debug,
857+
)
858+
if token is None:
859+
continue
860+
861+
token_line, token_column = token[0], token[1]
862+
delta_column = (
863+
token_column - column if token_line == line else token_column
864+
)
865+
delta_line = token_line - line
866+
867+
line = token_line
868+
column = token_column
869+
token[0] = delta_line
870+
token[1] = delta_column
871+
872+
server.show_message_log(
873+
f"diff token for name {n.description} ({n.line - 1}:{n.column}): {token}",
874+
MessageType.Debug,
875+
)
876+
data.extend(token)
877+
878+
return SemanticTokens(data=data)
879+
880+
727881
# Static capability or initializeOptions functions that rely on a specific
728882
# client capability or user configuration. These are associated with
729883
# JediLanguageServer within JediLanguageServerProtocol.lsp_initialize

‎tests/lsp_test_client/session.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,21 @@ def text_document_references(self, references_params):
191191
)
192192
return fut.result()
193193

194+
def text_doc_semantic_tokens_full(self, semantic_tokens_params):
195+
"""Sends text document semantic tokens full request to LSP server."""
196+
fut = self._send_request(
197+
"textDocument/semanticTokens/full", params=semantic_tokens_params
198+
)
199+
return fut.result()
200+
201+
def text_doc_semantic_tokens_range(self, semantic_tokens_range_params):
202+
"""Sends text document semantic tokens range request to LSP server."""
203+
fut = self._send_request(
204+
"textDocument/semanticTokens/range",
205+
params=semantic_tokens_range_params,
206+
)
207+
return fut.result()
208+
194209
def workspace_symbol(self, workspace_symbol_params):
195210
"""Sends workspace symbol request to LSP server."""
196211
fut = self._send_request(

‎tests/lsp_tests/test_semantic_tokens.py

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
"""Tests for semantic tokens requests."""
2+
3+
from hamcrest import assert_that, is_
4+
5+
from tests import TEST_DATA
6+
from tests.lsp_test_client import session
7+
from tests.lsp_test_client.utils import as_uri
8+
9+
SEMANTIC_TEST_ROOT = TEST_DATA / "semantic_tokens"
10+
11+
12+
def test_semantic_tokens_full_import():
13+
"""Tests tokens for 'import name1 as name2'.
14+
15+
Test Data: tests/test_data/semantic_tokens/semantic_tokens_test1.py.
16+
"""
17+
with session.LspSession() as ls_session:
18+
ls_session.initialize()
19+
uri = as_uri(SEMANTIC_TEST_ROOT / "semantic_tokens_test1.py")
20+
actual = ls_session.text_doc_semantic_tokens_full(
21+
{
22+
"textDocument": {"uri": uri},
23+
}
24+
)
25+
# fmt: off
26+
# [line, column, length, id, mod_id]
27+
expected = {
28+
"data": [
29+
5, 7, 2, 0, 0, # "import re"
30+
1, 7, 3, 0, 0, # "import sys, "
31+
0, 5, 2, 0, 0, # "os."
32+
0, 3, 4, 0, 0, # "path as "
33+
0, 8, 4, 0, 0, # "path"
34+
]
35+
}
36+
# fmt: on
37+
assert_that(actual, is_(expected))
38+
39+
40+
def test_semantic_tokens_full_import_from():
41+
"""Tests tokens for 'from name1 import name2 as name3'.
42+
43+
Test Data: tests/test_data/semantic_tokens/semantic_tokens_test2.py.
44+
"""
45+
with session.LspSession() as ls_session:
46+
ls_session.initialize()
47+
uri = as_uri(SEMANTIC_TEST_ROOT / "semantic_tokens_test2.py")
48+
actual = ls_session.text_doc_semantic_tokens_full(
49+
{
50+
"textDocument": {"uri": uri},
51+
}
52+
)
53+
# fmt: off
54+
# [line, column, length, id, mod_id]
55+
expected = {
56+
"data": [
57+
0, 5, 2, 0, 0, # "from os."
58+
0, 3, 4, 0, 0, # "path import "
59+
0, 12, 6, 12, 0, # "exists"
60+
1, 5, 3, 0, 0, # "from sys import"
61+
0, 11, 4, 8, 0, # "argv as "
62+
0, 8, 9, 8, 0, # "arguments"
63+
]
64+
}
65+
# fmt: on
66+
assert_that(actual, is_(expected))
67+
68+
69+
def test_semantic_tokens_range_import_from():
70+
"""Tests tokens for 'from name1 import name2 as name3'.
71+
72+
Test Data: tests/test_data/semantic_tokens/semantic_tokens_test2.py.
73+
"""
74+
with session.LspSession() as ls_session:
75+
ls_session.initialize()
76+
uri = as_uri(SEMANTIC_TEST_ROOT / "semantic_tokens_test2.py")
77+
actual = ls_session.text_doc_semantic_tokens_range(
78+
{
79+
"textDocument": {"uri": uri},
80+
"range": {
81+
"start": {"line": 1, "character": 1},
82+
"end": {"line": 2, "character": 0},
83+
},
84+
}
85+
)
86+
# fmt: off
87+
# [line, column, length, id, mod_id]
88+
expected = {
89+
"data": [
90+
0, 4, 3, 0, 0, # "from sys import"
91+
0, 11, 4, 8, 0, # "argv as "
92+
0, 8, 9, 8, 0, # "arguments"
93+
]
94+
}
95+
# fmt: on
96+
assert_that(actual, is_(expected))

‎tests/test_data/semantic_tokens/semantic_tokens_test1.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
"""Test file for semantic tokens import.
2+
3+
isort:skip_file
4+
"""
5+
6+
import re
7+
import sys, os.path as path

‎tests/test_data/semantic_tokens/semantic_tokens_test2.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
from os.path import exists
2+
from sys import argv as arguments

0 commit comments

Comments
 (0)