Log (instead of raise) exceptions when running as a server extension (jupyterlab#295)

* Log (instead of raise) exceptions when running as a server extension

* pre-commit errors

* Allow transient rooms to log exceptions

* Update projects/jupyter-server-ydoc/jupyter_server_ydoc/websocketserver.py

Co-authored-by: David Brochart <[email protected]>

---------

Co-authored-by: David Brochart <[email protected]>

Author: 2 people

projects/jupyter-server-ydoc/jupyter_server_ydoc/app.py

@@ -22,7 +22,7 @@
     encode_file_path,
     room_id_from_encoded_path,
 )
-from .websocketserver import JupyterWebsocketServer, RoomNotFound
+from .websocketserver import JupyterWebsocketServer, RoomNotFound, exception_logger
 
 
 class YDocExtension(ExtensionApp):
@@ -103,7 +103,15 @@ def initialize_handlers(self):
         for k, v in self.config.get(self.ystore_class.__name__, {}).items():
             setattr(self.ystore_class, k, v)
 
-        self.store = self.ystore_class(log=self.log)
+        self.ywebsocket_server = JupyterWebsocketServer(
+            rooms_ready=False,
+            auto_clean_rooms=False,
+            ystore_class=self.ystore_class,
+            # Log exceptions, because we don't want the websocket server
+            # to _ever_ crash permanently in a live jupyter_server.
+            exception_handler=exception_logger,
+            log=self.log,
+        )
 
         # self.settings is local to the ExtensionApp but here we need
         # the global app settings in which the file id manager will register

projects/jupyter-server-ydoc/jupyter_server_ydoc/handlers.py

@@ -6,6 +6,8 @@
 import asyncio
 import json
 import time
+import uuid
+from logging import Logger
 from typing import Any
 
 from jupyter_server.auth import authorized
@@ -84,6 +86,21 @@ async def prepare(self):
             if self._websocket_server.room_exists(self._room_id):
                 self.room: YRoom = await self._websocket_server.get_room(self._room_id)
             else:
+                # Logging exceptions, instead of raising them here to ensure
+                # that the y-rooms stay alive even after an exception is seen.
+                def exception_logger(exception: Exception, log: Logger) -> bool:
+                    """A function that catches any exceptions raised in the websocket
+                    server and logs them.
+
+                    The protects the y-room's task group from cancelling
+                    anytime an exception is raised.
+                    """
+                    log.error(
+                        f"Document Room Exception, (room_id={self._room_id or 'unknown'}): ",
+                        exc_info=exception,
+                    )
+                    return True
+
                 if self._room_id.count(":") >= 2:
                     # DocumentRoom
                     file_format, file_type, file_id = decode_file_path(self._room_id)
@@ -105,13 +122,18 @@ async def prepare(self):
                         self.event_logger,
                         ystore,
                         self.log,
-                        self._document_save_delay,
+                        exception_handler=exception_logger,
+                        save_delay=self._document_save_delay,
                     )
 
                 else:
                     # TransientRoom
                     # it is a transient document (e.g. awareness)
-                    self.room = TransientRoom(self._room_id, self.log)
+                    self.room = TransientRoom(
+                        self._room_id,
+                        log=self.log,
+                        exception_handler=exception_logger,
+                    )
 
             try:
                 await self._websocket_server.start_room(self.room)
@@ -223,12 +245,16 @@ async def open(self, room_id):
                 _, _, file_id = decode_file_path(self._room_id)
                 file = self._file_loaders[file_id]
 
-            # Clean up the room and delete the file loader
-            if self.room is not None and len(self.room.clients) == 0 or self.room.clients == [self]:
-                self._message_queue.put_nowait(b"")
-                if self._serve_task:
-                    self._serve_task.cancel()
-                await self._room_manager.remove_room(self._room_id)
+                # Close websocket and propagate error.
+                if isinstance(e, web.HTTPError):
+                    self.log.error(f"File {file.path} not found.\n{e!r}", exc_info=e)
+                    self.close(1004, f"File {file.path} not found.")
+                else:
+                    self.log.error(f"Error initializing: {file.path}\n{e!r}", exc_info=e)
+                    self.close(
+                        1003,
+                        f"Error initializing: {file.path}. You need to close the document.",
+                    )
 
             return
 
@@ -290,7 +316,11 @@ async def on_message(self, message):
 
             user = self.current_user
             data = json.dumps(
-                {"sender": user.username, "timestamp": time.time(), "content": json.loads(msg)}
+                {
+                    "sender": user.username,
+                    "timestamp": time.time(),
+                    "content": json.loads(msg),
+                }
             ).encode("utf8")
 
             self.room.broadcast_msg(bytes([MessageType.CHAT]) + write_var_uint(len(data)) + data)
@@ -414,6 +444,22 @@ async def put(self, path):
 
         status = 200
         idx = file_id_manager.get_id(path)
+        if idx is not None:
+            # index already exists
+            self.log.info("Request for Y document '%s' with room ID: %s", path, idx)
+            data = json.dumps(
+                {
+                    "format": format,
+                    "type": content_type,
+                    "fileId": idx,
+                    "sessionId": SERVER_SESSION,
+                }
+            )
+            self.set_status(200)
+            return self.finish(data)
+
+        # try indexing
+        idx = file_id_manager.index(path)
         if idx is None:
             # try indexing
             status = 201
@@ -426,7 +472,12 @@ async def put(self, path):
 
         self.log.info("Request for Y document '%s' with room ID: %s", path, idx)
         data = json.dumps(
-            {"format": format, "type": content_type, "fileId": idx, "sessionId": session_id}
+            {
+                "format": format,
+                "type": content_type,
+                "fileId": idx,
+                "sessionId": SERVER_SESSION,
+            }
         )
         self.set_status(status)
         return self.finish(data)

projects/jupyter-server-ydoc/jupyter_server_ydoc/rooms.py

@@ -5,7 +5,7 @@
 
 import asyncio
 from logging import Logger
-from typing import Any
+from typing import Any, Callable
 
 from jupyter_events import EventLogger
 from jupyter_ydoc import ydocs as YDOCS
@@ -31,8 +31,9 @@ def __init__(
         store: BaseYStore | None,
         log: Logger | None,
         save_delay: float | None = None,
+        exception_handler: Callable[[Exception, Logger], bool] | None = None,
     ):
-        super().__init__(room_id=room_id, store=store, log=log)
+        super().__init__(ready=False, ystore=ystore, exception_handler=exception_handler, log=log)
 
         self._file_format: str = file_format
         self._file_type: str = file_type
@@ -101,7 +102,9 @@ async def initialize(self) -> None:
                 if self._document.source != model["content"]:
                     # TODO: Delete document from the store.
                     self._emit(
-                        LogLevel.INFO, "initialize", "The file is out-of-sync with the ystore."
+                        LogLevel.INFO,
+                        "initialize",
+                        "The file is out-of-sync with the ystore.",
                     )
                     self.log.info(
                         "Content in file %s is out-of-sync with the ystore %s",
@@ -113,7 +116,9 @@ async def initialize(self) -> None:
             if read_from_source:
                 self._emit(LogLevel.INFO, "load", "Content loaded from disk.")
                 self.log.info(
-                    "Content in room %s loaded from file %s", self._room_id, self._file.path
+                    "Content in room %s loaded from file %s",
+                    self._room_id,
+                    self._file.path,
                 )
                 self._document.source = model["content"]
 
@@ -266,8 +271,13 @@ async def _maybe_save_document(self, saving_document: asyncio.Task | None) -> No
 class TransientRoom(YRoom):
     """A Y room for sharing state (e.g. awareness)."""
 
-    def __init__(self, room_id: str, log: Logger | None):
-        super().__init__(log=log)
+    def __init__(
+        self,
+        room_id: str,
+        log: Logger | None = None,
+        exception_handler: Callable[[Exception, Logger], bool] | None = None,
+    ):
+        super().__init__(log=log, exception_handler=exception_handler)
 
         self._room_id = room_id
 

projects/jupyter-server-ydoc/jupyter_server_ydoc/websocketserver.py

@@ -5,7 +5,7 @@
 
 import asyncio
 from logging import Logger
-from typing import Any
+from typing import Any, Callable
 
 from pycrdt_websocket.websocket_server import WebsocketServer, YRoom
 from pycrdt_websocket.ystore import BaseYStore
@@ -16,6 +16,17 @@ class RoomNotFound(LookupError):
     pass
 
 
+def exception_logger(exception: Exception, log: Logger) -> bool:
+    """A function that catches any exceptions raised in the websocket
+    server and logs them.
+
+    This protects the websocket server's task group from cancelling
+    anytime an exception is raised.
+    """
+    log.error("Jupyter Websocket Server: ", exc_info=exception)
+    return True
+
+
 class JupyterWebsocketServer(WebsocketServer):
     """Ypy websocket server.
 
@@ -29,9 +40,16 @@ def __init__(
         self,
         rooms_ready: bool = True,
         auto_clean_rooms: bool = True,
+        exception_handler: Callable[[Exception, Logger], bool] | None = None,
         log: Logger | None = None,
     ):
-        super().__init__(rooms_ready, auto_clean_rooms, log)
+        super().__init__(
+            rooms_ready=rooms_ready,
+            auto_clean_rooms=auto_clean_rooms,
+            exception_handler=exception_handler,
+            log=log,
+        )
+        self.ystore_class = ystore_class
         self.ypatch_nb = 0
         self.connected_users: dict[Any, Any] = {}
         # Async loop is not yet ready at the object instantiation

projects/jupyter-server-ydoc/pyproject.toml

@@ -30,7 +30,7 @@ authors = [
 dependencies = [
     "jupyter_server>=2.4.0,<3.0.0",
     "jupyter_ydoc>=2.0.0,<3.0.0",
-    "pycrdt-websocket>=0.13.0,<0.14.0",
+    "pycrdt-websocket>=0.13.1,<0.14.0",
     "jupyter_events>=0.10.0",
     "jupyter_server_fileid>=0.7.0,<1",
     "jsonschema>=4.18.0"