Merge branch 'master' into backend-status

Author: krassowski

CHANGELOG.md

@@ -9,24 +9,28 @@
 
 - bug fixes:
 
-  - fix completions with R double and triple colon prefix ([#449])
-  - fix contrast on status icon when status item is active ([#465])
-  - fix connection manager loosing track of notebooks when multiple were open ([#474])
+  - namespace completions in R (after double and triple colon) now work properly ([#449])
+  - improved status icon contrast when status item is active ([#465])
+  - connection manager now properly keeps track of notebooks when multiple notebooks are open ([#474])
+  - new cells added after kernel restart now work properly; kernel changes are handled correctly ([#478])
 
 [#449]: https://github.com/krassowski/jupyterlab-lsp/pull/449
 [#465]: https://github.com/krassowski/jupyterlab-lsp/pull/465
 [#474]: https://github.com/krassowski/jupyterlab-lsp/pull/474
 [#476]: https://github.com/krassowski/jupyterlab-lsp/pull/476
+[#478]: https://github.com/krassowski/jupyterlab-lsp/pull/478
 
 ### `jupyter-lsp 1.0.1` (unreleased)
 
 - bug fixes:
 
   - send periodic pings on websocket channels to maintain connection ([#459], thanks @franckchen)
   - R languageserver is no longer incorrectly shown as available when not installed ([#463])
+  - fix completion of very large namespaces (e.g. in R's base or in JavaScript) due to truncated message relay ([#477])
 
 [#459]: https://github.com/krassowski/jupyterlab-lsp/pull/459
 [#463]: https://github.com/krassowski/jupyterlab-lsp/pull/463
+[#477]: https://github.com/krassowski/jupyterlab-lsp/pull/477
 
 ### `@krassowski/jupyterlab-lsp 3.0.0` (2021-01-06)
 

atest/05_Features/Completion.robot

@@ -28,6 +28,7 @@ Works With Kernel Running
 
 Works When Kernel Is Shut Down
     Lab Command    Shut Down All Kernels…
+    Wait For Dialog
     Capture Page Screenshot    01-shutting-kernels.png
     Accept Default Dialog Option
     Capture Page Screenshot    02-kernels-shut.png
@@ -39,6 +40,21 @@ Works When Kernel Is Shut Down
     # this comes from kernel:
     Completer Should Not Suggest    %%timeit
 
+Works After Kernel Restart In New Cells
+    Lab Command    Restart Kernel…
+    Wait For Dialog
+    Accept Default Dialog Option
+    Enter Cell Editor    1    line=2
+    # works in old cells
+    Trigger Completer
+    Completer Should Suggest    test
+    Lab Command    Insert Cell Below
+    Enter Cell Editor    2    line=1
+    # works in new cells
+    Press Keys    None    lis
+    Trigger Completer
+    Completer Should Suggest    list
+
 Works In File Editor
     [Setup]    Prepare File for Editing    Python    completion    completion.py
     Place Cursor In File Editor At    9    2
@@ -203,6 +219,14 @@ Completes Correctly With R Double And Triple Colon
     Wait Until Keyword Succeeds    40x    0.5s    File Editor Line Should Equal    3    datasets:::.packageName
     [Teardown]    Clean Up After Working With File    completion.R
 
+Completes Large Namespaces
+    [Setup]    Prepare File for Editing    R    completion    completion.R
+    Place Cursor In File Editor At    6    7
+    Wait Until Fully Initialized
+    Trigger Completer
+    Completer Should Suggest    abs    timeout=30s
+    [Teardown]    Clean Up After Working With File    completion.R
+
 *** Keywords ***
 Setup Completion Test
     Setup Notebook    Python    Completion.ipynb
@@ -235,8 +259,8 @@ Select Completer Suggestion
     Click Element    ${suggestion} code
 
 Completer Should Suggest
-    [Arguments]    ${text}
-    Wait Until Page Contains Element    ${COMPLETER_BOX} .jp-Completer-item[data-value="${text}"]    timeout=10s
+    [Arguments]    ${text}    ${timeout}=10s
+    Wait Until Page Contains Element    ${COMPLETER_BOX} .jp-Completer-item[data-value="${text}"]    timeout=${timeout}
     Capture Page Screenshot    ${text.replace(' ', '_')}.png
 
 Completer Should Include Icon

atest/examples/completion.R

@@ -2,3 +2,5 @@
 tools::
 # `datasets:::<tab>` → select `.packageName` → `datasets:::.packageName`
 datasets:::
+# `base:::<tab>` → works
+base:::

docs/Configuring.ipynb

@@ -210,6 +210,9 @@
   },
   {
    "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false
+   },
    "source": [
     "### Example: Scala Language Server (metals) integration with jupyterlab-lsp\n",
     "\n",
@@ -225,11 +228,13 @@
     "$ ./coursier launch --fork almond -- --install\n",
     "$ rm -f coursier\n",
     "```\n",
+    "\n",
     "Spark Magic kernel:\n",
     "\n",
     "```bash\n",
     "pip install sparkmagic\n",
     "```\n",
+    "\n",
     "Now, install the spark kernel:\n",
     "\n",
     "```bash\n",
@@ -247,7 +252,8 @@
     "\n",
     "(Might need to use the --force-fetch flag if you are getting dependency issues.)\n",
     "\n",
-    "Step 3: Configure the metals server in jupyterlab-lsp. Enter the following in the jupyter_server_config.json:\n",
+    "Step 3: Configure the metals server in jupyterlab-lsp. Enter the following in\n",
+    "the jupyter_server_config.json:\n",
     "\n",
     "```python\n",
     "{\n",
@@ -264,11 +270,10 @@
     "}\n",
     "```\n",
     "\n",
-    "You are good to go now! Just start `jupyter lab` and create a notebook with either the Spark or the Scala kernel and you should be able to see the metals server initialised from the bottom left corner."
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+    "You are good to go now! Just start `jupyter lab` and create a notebook with\n",
+    "either the Spark or the Scala kernel and you should be able to see the metals\n",
+    "server initialised from the bottom left corner."
+   ]
   }
  ],
  "metadata": {

packages/jupyterlab-lsp/src/adapters/adapter.ts

@@ -236,10 +236,17 @@ export abstract class WidgetAdapter<T extends IDocumentWidget> {
     );
 
     // recreate virtual document using current path and language
-    let virtual_document = this.create_virtual_document();
-    this.virtual_editor.virtual_document = virtual_document;
+    // as virtual editor assumes it gets the virtual document at init,
+    // just dispose virtual editor (which disposes virtual document too)
+    // and re-initialize both virtual editor and document
+    this.virtual_editor.dispose();
+
+    this.init_virtual();
+
     // reconnect
-    this.connect_document(virtual_document, true).catch(console.warn);
+    this.connect_document(this.virtual_editor.virtual_document, true).catch(
+      console.warn
+    );
   }
 
   protected on_save_state(context: any, state: DocumentRegistry.SaveState) {

python_packages/jupyter_lsp/jupyter_lsp/specs/utils.py

@@ -67,12 +67,12 @@ def __call__(self, mgr: LanguageServerManagerAPI) -> KeyedLanguageServerSpecs:
             if cmd:
                 break
 
-        if not cmd and BUILDING_DOCS:  # pragma: no cover
-            cmd = self.cmd
-
         if not self.is_installed(cmd):  # pragma: no cover
             return {}
 
+        if not cmd and BUILDING_DOCS:  # pragma: no cover
+            cmd = self.cmd
+
         return {
             self.key: {
                 "argv": [cmd, *self.args],

python_packages/jupyter_lsp/jupyter_lsp/stdio.py

@@ -12,7 +12,7 @@
 import io
 import os
 from concurrent.futures import ThreadPoolExecutor
-from typing import Text
+from typing import List, Optional, Text
 
 from tornado.concurrent import run_on_executor
 from tornado.gen import convert_yielded
@@ -30,7 +30,9 @@ class LspStdIoBase(LoggingConfigurable):
 
     executor = None
 
-    stream = Instance(io.BufferedIOBase, help="the stream to read/write")
+    stream = Instance(
+        io.BufferedIOBase, help="the stream to read/write"
+    )  # type: io.BufferedIOBase
     queue = Instance(Queue, help="queue to get/put")
 
     def __repr__(self):  # pragma: no cover
@@ -95,6 +97,39 @@ async def read(self) -> None:
                 self.log.exception("%s couldn't enqueue message: %s", self, message)
                 await self.sleep()
 
+    def _read_content(self, length: int, max_parts=1000) -> Optional[bytes]:
+        """Read the full length of the message unless exceeding max_parts.
+
+        See https://github.com/krassowski/jupyterlab-lsp/issues/450
+
+        Crucial docs or read():
+            "If the argument is positive, and the underlying raw
+             stream is not interactive, multiple raw reads may be issued
+             to satisfy the byte count (unless EOF is reached first)"
+
+        Args:
+           - length: the content length
+           - max_parts: prevent absurdly long messages (1000 parts is several MBs):
+             1 part is usually sufficent but not enough for some long
+             messages 2 or 3 parts are often needed.
+        """
+        raw_parts: List[bytes] = []
+        received_size = 0
+        while received_size < length and len(raw_parts) < max_parts:
+            part = self.stream.read(length)
+            if part is None:
+                break  # pragma: no cover
+            received_size += len(part)
+            raw_parts.append(part)
+
+        if raw_parts:
+            raw = b"".join(raw_parts)
+            if len(raw) != length:  # pragma: no cover
+                self.log.warning(
+                    f"Readout and content-length mismatch:" f" {len(raw)} vs {length}"
+                )
+        return raw
+
     async def read_one(self) -> Text:
         """Read a single message"""
         message = ""
@@ -114,7 +149,7 @@ async def read_one(self) -> Text:
                 retries = 5
                 while raw is None and retries:
                     try:
-                        raw = self.stream.read(content_length)
+                        raw = self._read_content(length=content_length)
                     except OSError:  # pragma: no cover
                         raw = None
                     if raw is None:  # pragma: no cover