Merge pull request #477 from krassowski/fix-truncated-stream

Fix truncated stream

Author: krassowski

CHANGELOG.md

@@ -22,9 +22,11 @@
 
   - send periodic pings on websocket channels to maintain connection ([#459], thanks @franckchen)
   - R languageserver is no longer incorrectly shown as available when not installed ([#463])
+  - fix completion of very large namespaces (e.g. in R's base or in JavaScript) due to truncated message relay ([#477])
 
 [#459]: https://github.com/krassowski/jupyterlab-lsp/pull/459
 [#463]: https://github.com/krassowski/jupyterlab-lsp/pull/463
+[#477]: https://github.com/krassowski/jupyterlab-lsp/pull/477
 
 ### `@krassowski/jupyterlab-lsp 3.0.0` (2021-01-06)
 

atest/05_Features/Completion.robot

@@ -203,6 +203,14 @@ Completes Correctly With R Double And Triple Colon
     Wait Until Keyword Succeeds    40x    0.5s    File Editor Line Should Equal    3    datasets:::.packageName
     [Teardown]    Clean Up After Working With File    completion.R
 
+Completes Large Namespaces
+    [Setup]    Prepare File for Editing    R    completion    completion.R
+    Place Cursor In File Editor At    6    7
+    Wait Until Fully Initialized
+    Trigger Completer
+    Completer Should Suggest    abs    timeout=30s
+    [Teardown]    Clean Up After Working With File    completion.R
+
 *** Keywords ***
 Setup Completion Test
     Setup Notebook    Python    Completion.ipynb
@@ -235,8 +243,8 @@ Select Completer Suggestion
     Click Element    ${suggestion} code
 
 Completer Should Suggest
-    [Arguments]    ${text}
-    Wait Until Page Contains Element    ${COMPLETER_BOX} .jp-Completer-item[data-value="${text}"]    timeout=10s
+    [Arguments]    ${text}    ${timeout}=10s
+    Wait Until Page Contains Element    ${COMPLETER_BOX} .jp-Completer-item[data-value="${text}"]    timeout=${timeout}
     Capture Page Screenshot    ${text.replace(' ', '_')}.png
 
 Completer Should Include Icon

atest/examples/completion.R

@@ -2,3 +2,5 @@
 tools::
 # `datasets:::<tab>` → select `.packageName` → `datasets:::.packageName`
 datasets:::
+# `base:::<tab>` → works
+base:::

docs/Configuring.ipynb

@@ -210,6 +210,9 @@
   },
   {
    "cell_type": "markdown",
+   "metadata": {
+    "collapsed": false
+   },
    "source": [
     "### Example: Scala Language Server (metals) integration with jupyterlab-lsp\n",
     "\n",
@@ -225,11 +228,13 @@
     "$ ./coursier launch --fork almond -- --install\n",
     "$ rm -f coursier\n",
     "```\n",
+    "\n",
     "Spark Magic kernel:\n",
     "\n",
     "```bash\n",
     "pip install sparkmagic\n",
     "```\n",
+    "\n",
     "Now, install the spark kernel:\n",
     "\n",
     "```bash\n",
@@ -247,7 +252,8 @@
     "\n",
     "(Might need to use the --force-fetch flag if you are getting dependency issues.)\n",
     "\n",
-    "Step 3: Configure the metals server in jupyterlab-lsp. Enter the following in the jupyter_server_config.json:\n",
+    "Step 3: Configure the metals server in jupyterlab-lsp. Enter the following in\n",
+    "the jupyter_server_config.json:\n",
     "\n",
     "```python\n",
     "{\n",
@@ -264,11 +270,10 @@
     "}\n",
     "```\n",
     "\n",
-    "You are good to go now! Just start `jupyter lab` and create a notebook with either the Spark or the Scala kernel and you should be able to see the metals server initialised from the bottom left corner."
-   ],
-   "metadata": {
-    "collapsed": false
-   }
+    "You are good to go now! Just start `jupyter lab` and create a notebook with\n",
+    "either the Spark or the Scala kernel and you should be able to see the metals\n",
+    "server initialised from the bottom left corner."
+   ]
   }
  ],
  "metadata": {

python_packages/jupyter_lsp/jupyter_lsp/stdio.py

@@ -12,7 +12,7 @@
 import io
 import os
 from concurrent.futures import ThreadPoolExecutor
-from typing import Text
+from typing import List, Optional, Text
 
 from tornado.concurrent import run_on_executor
 from tornado.gen import convert_yielded
@@ -30,7 +30,9 @@ class LspStdIoBase(LoggingConfigurable):
 
     executor = None
 
-    stream = Instance(io.BufferedIOBase, help="the stream to read/write")
+    stream = Instance(
+        io.BufferedIOBase, help="the stream to read/write"
+    )  # type: io.BufferedIOBase
     queue = Instance(Queue, help="queue to get/put")
 
     def __repr__(self):  # pragma: no cover
@@ -95,6 +97,39 @@ async def read(self) -> None:
                 self.log.exception("%s couldn't enqueue message: %s", self, message)
                 await self.sleep()
 
+    def _read_content(self, length: int, max_parts=1000) -> Optional[bytes]:
+        """Read the full length of the message unless exceeding max_parts.
+
+        See https://github.com/krassowski/jupyterlab-lsp/issues/450
+
+        Crucial docs or read():
+            "If the argument is positive, and the underlying raw
+             stream is not interactive, multiple raw reads may be issued
+             to satisfy the byte count (unless EOF is reached first)"
+
+        Args:
+           - length: the content length
+           - max_parts: prevent absurdly long messages (1000 parts is several MBs):
+             1 part is usually sufficent but not enough for some long
+             messages 2 or 3 parts are often needed.
+        """
+        raw_parts: List[bytes] = []
+        received_size = 0
+        while received_size < length and len(raw_parts) < max_parts:
+            part = self.stream.read(length)
+            if part is None:
+                break  # pragma: no cover
+            received_size += len(part)
+            raw_parts.append(part)
+
+        if raw_parts:
+            raw = b"".join(raw_parts)
+            if len(raw) != length:  # pragma: no cover
+                self.log.warning(
+                    f"Readout and content-length mismatch:" f" {len(raw)} vs {length}"
+                )
+        return raw
+
     async def read_one(self) -> Text:
         """Read a single message"""
         message = ""
@@ -114,7 +149,7 @@ async def read_one(self) -> Text:
                 retries = 5
                 while raw is None and retries:
                     try:
-                        raw = self.stream.read(content_length)
+                        raw = self._read_content(length=content_length)
                     except OSError:  # pragma: no cover
                         raw = None
                     if raw is None:  # pragma: no cover