From 93f7fa6ba8df3c0beba6911b639ea6f5bed106c1 Mon Sep 17 00:00:00 2001 From: Zach Sailer Date: Tue, 23 Apr 2024 17:11:09 -0700 Subject: [PATCH 1/8] Add async start hook to ExtensionApp API --- jupyter_server/extension/application.py | 3 + jupyter_server/extension/manager.py | 14 +++ jupyter_server/serverapp.py | 120 +++++++++++++----------- tests/extension/mockextensions/app.py | 6 +- tests/extension/test_app.py | 5 + tests/test_serverapp.py | 3 +- 6 files changed, 95 insertions(+), 56 deletions(-) diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 32ac94cfcb..73067c6659 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -446,6 +446,9 @@ def start(self): assert self.serverapp is not None self.serverapp.start() + async def start_extension(self): + """An async hook to start e.g. tasks after the server's event loop is running.""" + def current_activity(self): """Return a list of activity happening in this extension.""" return diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index b8c52ca9e5..4cdcb9d9a6 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -367,6 +367,16 @@ def load_extension(self, name): else: self.log.info("%s | extension was successfully loaded.", name) + async def start_extension(self, name, apps): + """Call the start hooks in the specified apps.""" + for app in apps: + self.log.debug("%s | extension app %r starting", name, app.name) + try: + await app.start_extension() + self.log.debug("%s | extension app %r started", name, app.name) + except Exception as err: + self.log.error(err) + async def stop_extension(self, name, apps): """Call the shutdown hooks in the specified apps.""" for app in apps: @@ -392,6 +402,10 @@ def load_all_extensions(self): for name in self.sorted_extensions: self.load_extension(name) + async def start_all_extensions(self): + """Call the start hooks in all extensions.""" + await multi(list(starmap(self.start_extension, sorted(dict(self.extension_apps).items())))) + async def stop_all_extensions(self): """Call the shutdown hooks in all extensions.""" await multi(list(starmap(self.stop_extension, sorted(dict(self.extension_apps).items())))) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 13fa256397..8e19658fc5 100644 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -3061,6 +3061,72 @@ def start_app(self) -> None: ) self.exit(1) + self.write_server_info_file() + + if not self.no_browser_open_file: + self.write_browser_open_files() + + # Handle the browser opening. + if self.open_browser and not self.sock: + self.launch_browser() + + async def _cleanup(self) -> None: + """General cleanup of files, extensions and kernels created + by this instance ServerApp. + """ + self.remove_server_info_file() + self.remove_browser_open_files() + await self.cleanup_extensions() + await self.cleanup_kernels() + try: + await self.kernel_websocket_connection_class.close_all() # type:ignore[attr-defined] + except AttributeError: + # This can happen in two different scenarios: + # + # 1. During tests, where the _cleanup method is invoked without + # the corresponding initialize method having been invoked. + # 2. If the provided `kernel_websocket_connection_class` does not + # implement the `close_all` class method. + # + # In either case, we don't need to do anything and just want to treat + # the raised error as a no-op. + pass + if getattr(self, "kernel_manager", None): + self.kernel_manager.__del__() + if getattr(self, "session_manager", None): + self.session_manager.close() + if hasattr(self, "http_server"): + # Stop a server if its set. + self.http_server.stop() + + def start_ioloop(self) -> None: + """Start the IO Loop.""" + if sys.platform.startswith("win"): + # add no-op to wake every 5s + # to handle signals that may be ignored by the inner loop + pc = ioloop.PeriodicCallback(lambda: None, 5000) + pc.start() + try: + self.io_loop.add_callback(self._post_start) + self.io_loop.start() + except KeyboardInterrupt: + self.log.info(_i18n("Interrupted...")) + + def init_ioloop(self) -> None: + """init self.io_loop so that an extension can use it by io_loop.call_later() to create background tasks""" + self.io_loop = ioloop.IOLoop.current() + + async def _post_start(self): + """Add an async hook to start tasks after the event loop is running. + + This will also attempt to start all tasks found in + the `start_extension` method in Extension Apps. + """ + try: + await self.extension_manager.start_all_extensions() + except Exception as err: + self.log.error(err) + info = self.log.info for line in self.running_server_info(kernel_count=False).split("\n"): info(line) @@ -3079,15 +3145,6 @@ def start_app(self) -> None: ) ) - self.write_server_info_file() - - if not self.no_browser_open_file: - self.write_browser_open_files() - - # Handle the browser opening. - if self.open_browser and not self.sock: - self.launch_browser() - if self.identity_provider.token and self.identity_provider.token_generated: # log full URL with generated token, so there's a copy/pasteable link # with auth info. @@ -3128,51 +3185,6 @@ def start_app(self) -> None: self.log.critical("\n".join(message)) - async def _cleanup(self) -> None: - """General cleanup of files, extensions and kernels created - by this instance ServerApp. - """ - self.remove_server_info_file() - self.remove_browser_open_files() - await self.cleanup_extensions() - await self.cleanup_kernels() - try: - await self.kernel_websocket_connection_class.close_all() # type:ignore[attr-defined] - except AttributeError: - # This can happen in two different scenarios: - # - # 1. During tests, where the _cleanup method is invoked without - # the corresponding initialize method having been invoked. - # 2. If the provided `kernel_websocket_connection_class` does not - # implement the `close_all` class method. - # - # In either case, we don't need to do anything and just want to treat - # the raised error as a no-op. - pass - if getattr(self, "kernel_manager", None): - self.kernel_manager.__del__() - if getattr(self, "session_manager", None): - self.session_manager.close() - if hasattr(self, "http_server"): - # Stop a server if its set. - self.http_server.stop() - - def start_ioloop(self) -> None: - """Start the IO Loop.""" - if sys.platform.startswith("win"): - # add no-op to wake every 5s - # to handle signals that may be ignored by the inner loop - pc = ioloop.PeriodicCallback(lambda: None, 5000) - pc.start() - try: - self.io_loop.start() - except KeyboardInterrupt: - self.log.info(_i18n("Interrupted...")) - - def init_ioloop(self) -> None: - """init self.io_loop so that an extension can use it by io_loop.call_later() to create background tasks""" - self.io_loop = ioloop.IOLoop.current() - def start(self) -> None: """Start the Jupyter server app, after initialization diff --git a/tests/extension/mockextensions/app.py b/tests/extension/mockextensions/app.py index 26f38464cd..52248d8dc7 100644 --- a/tests/extension/mockextensions/app.py +++ b/tests/extension/mockextensions/app.py @@ -4,7 +4,7 @@ from jupyter_events import EventLogger from jupyter_events.schema_registry import SchemaRegistryException -from traitlets import List, Unicode +from traitlets import Bool, List, Unicode from jupyter_server.base.handlers import JupyterHandler from jupyter_server.extension.application import ExtensionApp, ExtensionAppJinjaMixin @@ -50,6 +50,7 @@ class MockExtensionApp(ExtensionAppJinjaMixin, ExtensionApp): static_paths = [STATIC_PATH] # type:ignore[assignment] mock_trait = Unicode("mock trait", config=True) loaded = False + started = Bool(False) serverapp_config = {"jpserver_extensions": {"tests.extension.mockextensions.mock1": True}} @@ -71,6 +72,9 @@ def initialize_handlers(self): self.handlers.append(("/mock_template", MockExtensionTemplateHandler)) self.loaded = True + async def start_extension(self): + self.started = True + if __name__ == "__main__": MockExtensionApp.launch_instance() diff --git a/tests/extension/test_app.py b/tests/extension/test_app.py index d1add54344..7e1671869d 100644 --- a/tests/extension/test_app.py +++ b/tests/extension/test_app.py @@ -139,6 +139,11 @@ async def test_load_parallel_extensions(monkeypatch, jp_environ): assert exts["tests.extension.mockextensions"] +async def test_start_extension(jp_serverapp, mock_extension): + await jp_serverapp._post_start() + assert mock_extension.started + + async def test_stop_extension(jp_serverapp, caplog): """Test the stop_extension method. diff --git a/tests/test_serverapp.py b/tests/test_serverapp.py index 91fa33230c..78efcc9d4e 100644 --- a/tests/test_serverapp.py +++ b/tests/test_serverapp.py @@ -606,8 +606,9 @@ def test_running_server_info(jp_serverapp): @pytest.mark.parametrize("should_exist", [True, False]) -def test_browser_open_files(jp_configurable_serverapp, should_exist, caplog): +async def test_browser_open_files(jp_configurable_serverapp, should_exist, caplog): app = jp_configurable_serverapp(no_browser_open_file=not should_exist) + await app._post_start() assert os.path.exists(app.browser_open_file) == should_exist url = urljoin("file:", pathname2url(app.browser_open_file)) url_messages = [rec.message for rec in caplog.records if url in rec.message] From d3e65249052a1e3d7e4402daa6666f3fd338b458 Mon Sep 17 00:00:00 2001 From: Zach Sailer Date: Thu, 25 Apr 2024 12:31:19 -0700 Subject: [PATCH 2/8] add start hook to basic extensions and include documentation --- docs/source/developers/extensions.rst | 66 +++++++++++++++++++++++++ jupyter_server/extension/application.py | 15 ++++-- jupyter_server/extension/manager.py | 62 ++++++++++++++++++----- tests/extension/mockextensions/app.py | 2 +- tests/extension/mockextensions/mock1.py | 7 +++ tests/extension/test_app.py | 4 ++ 6 files changed, 141 insertions(+), 15 deletions(-) diff --git a/docs/source/developers/extensions.rst b/docs/source/developers/extensions.rst index be454b26e6..363795faf4 100644 --- a/docs/source/developers/extensions.rst +++ b/docs/source/developers/extensions.rst @@ -65,6 +65,29 @@ Then add this handler to Jupyter Server's Web Application through the ``_load_ju serverapp.web_app.add_handlers(".*$", handlers) +Starting asynchronous tasks from an extension +--------------------------------------------- + +.. versionadded:: 2.15.0 + +Jupyter Server offers a simple API for starting asynchronous tasks from a server extension. This is useful for calling +async tasks after the event loop is running. + +The function should be named ``_start_jupyter_server_extension`` and found next to the ``_load_jupyter_server_extension`` function. + +Here is basic example: + +.. code-block:: python + + import asyncio + + async def _start_jupyter_server_extension(serverapp: jupyter_server.serverapp.ServerApp): + """ + This function is called after the server's event loop is running. + """ + await asyncio.sleep(.1) + + Making an extension discoverable -------------------------------- @@ -117,6 +140,7 @@ An ExtensionApp: - has an entrypoint, ``jupyter ``. - can serve static content from the ``/static//`` endpoint. - can add new endpoints to the Jupyter Server. + - can start asynchronous tasks after the server has started. The basic structure of an ExtensionApp is shown below: @@ -156,6 +180,11 @@ The basic structure of an ExtensionApp is shown below: ... # Change the jinja templating environment + async def _start_jupyter_server_extension(self): + ... + # Extend this method to start any (e.g. async) tasks + # after the main Server's Event Loop is running. + async def stop_extension(self): ... # Perform any required shut down steps @@ -171,6 +200,7 @@ Methods * ``initialize_settings()``: adds custom settings to the Tornado Web Application. * ``initialize_handlers()``: appends handlers to the Tornado Web Application. * ``initialize_templates()``: initialize the templating engine (e.g. jinja2) for your frontend. +* ``_start_jupyter_server_extension()``: enables the extension to start (async) tasks _after_ the server's main Event Loop has started. * ``stop_extension()``: called on server shut down. Properties @@ -320,6 +350,42 @@ pointing at the ``load_classic_server_extension`` method: If the extension is enabled, the extension will be loaded when the server starts. +Starting asynchronous tasks from an ExtensionApp +------------------------------------------------ + +.. versionadded:: 2.15.0 + + +An ``ExtensionApp`` can start asynchronous tasks after Jupyter Server's event loop is started by overriding its ``_start_jupyter_server_extension()`` method. This can be helpful for setting up e.g. background tasks. + +Here is a basic (pseudo) code example: + +.. code-block:: python + + import asyncio + import time + + + async def log_time_periodically(log, dt=1): + """Log the current time from a periodic loop.""" + while True: + current_time = time.time() + log.info(current_time) + await sleep(dt) + + + class MyExtension(ExtensionApp): + ... + + async def _start_jupyter_server_extension(self): + self.my_background_task = asyncio.create_task( + log_time_periodically(self.log) + ) + + async def stop_extension(self): + self.my_background_task.cancel() + + Distributing a server extension =============================== diff --git a/jupyter_server/extension/application.py b/jupyter_server/extension/application.py index 73067c6659..698d920801 100644 --- a/jupyter_server/extension/application.py +++ b/jupyter_server/extension/application.py @@ -446,9 +446,6 @@ def start(self): assert self.serverapp is not None self.serverapp.start() - async def start_extension(self): - """An async hook to start e.g. tasks after the server's event loop is running.""" - def current_activity(self): """Return a list of activity happening in this extension.""" return @@ -478,6 +475,18 @@ def _load_jupyter_server_extension(cls, serverapp): extension.initialize() return extension + async def _start_jupyter_server_extension(self, serverapp): + """ + An async hook to start e.g. tasks from the extension after + the server's event loop is running. + + Override this method (no need to call `super()`) to + start (async) tasks from an extension. + + This is useful for starting e.g. background tasks from + an extension. + """ + @classmethod def load_classic_server_extension(cls, serverapp): """Enables extension to be loaded as classic Notebook (jupyter/notebook) extension.""" diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 4cdcb9d9a6..f51be19ed7 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -119,6 +119,20 @@ def _get_loader(self): loader = get_loader(loc) return loader + def _get_starter(self): + """Get a linker.""" + if self.app: + linker = self.app._start_jupyter_server_extension + else: + linker = getattr( + self.module, + # Search for a _start_jupyter_extension + "_start_jupyter_server_extension", + # Otherwise return a dummy function. + lambda serverapp: None, + ) + return linker + def validate(self): """Check that both a linker and loader exists.""" try: @@ -150,6 +164,13 @@ def load(self, serverapp): loader = self._get_loader() return loader(serverapp) + def start(self, serverapp): + """Call's the extensions 'start' hook where it can + start (possibly async) tasks _after_ the event loop is running. + """ + starter = self._get_starter() + return starter(serverapp) + class ExtensionPackage(LoggingConfigurable): """An API for interfacing with a Jupyter Server extension package. @@ -222,6 +243,11 @@ def load_point(self, point_name, serverapp): point = self.extension_points[point_name] return point.load(serverapp) + def start_point(self, point_name, serverapp): + """Load an extension point.""" + point = self.extension_points[point_name] + return point.start(serverapp) + def link_all_points(self, serverapp): """Link all extension points.""" for point_name in self.extension_points: @@ -231,9 +257,14 @@ def load_all_points(self, serverapp): """Load all extension points.""" return [self.load_point(point_name, serverapp) for point_name in self.extension_points] + async def start_all_points(self, serverapp): + """Load all extension points.""" + for point_name in self.extension_points: + await self.start_point(point_name, serverapp) + class ExtensionManager(LoggingConfigurable): - """High level interface for findind, validating, + """High level interface for finding, validating, linking, loading, and managing Jupyter Server extensions. Usage: @@ -367,15 +398,21 @@ def load_extension(self, name): else: self.log.info("%s | extension was successfully loaded.", name) - async def start_extension(self, name, apps): - """Call the start hooks in the specified apps.""" - for app in apps: - self.log.debug("%s | extension app %r starting", name, app.name) + async def start_extension(self, name): + """Start an extension by name.""" + extension = self.extensions.get(name) + + if extension and extension.enabled: try: - await app.start_extension() - self.log.debug("%s | extension app %r started", name, app.name) - except Exception as err: - self.log.error(err) + await extension.start_all_points(self.serverapp) + except Exception as e: + if self.serverapp and self.serverapp.reraise_server_extension_failures: + raise + self.log.warning( + "%s | extension failed starting with message: %r", name, e, exc_info=True + ) + else: + self.log.info("%s | extension was successfully started.", name) async def stop_extension(self, name, apps): """Call the shutdown hooks in the specified apps.""" @@ -403,8 +440,11 @@ def load_all_extensions(self): self.load_extension(name) async def start_all_extensions(self): - """Call the start hooks in all extensions.""" - await multi(list(starmap(self.start_extension, sorted(dict(self.extension_apps).items())))) + """Start all enabled extensions.""" + # Sort the extension names to enforce deterministic loading + # order. + for name in self.sorted_extensions: + await self.start_extension(name) async def stop_all_extensions(self): """Call the shutdown hooks in all extensions.""" diff --git a/tests/extension/mockextensions/app.py b/tests/extension/mockextensions/app.py index 52248d8dc7..e71824fc90 100644 --- a/tests/extension/mockextensions/app.py +++ b/tests/extension/mockextensions/app.py @@ -72,7 +72,7 @@ def initialize_handlers(self): self.handlers.append(("/mock_template", MockExtensionTemplateHandler)) self.loaded = True - async def start_extension(self): + async def _start_jupyter_server_extension(self, serverapp): self.started = True diff --git a/tests/extension/mockextensions/mock1.py b/tests/extension/mockextensions/mock1.py index 6113ab2fc5..bd73cdfa6a 100644 --- a/tests/extension/mockextensions/mock1.py +++ b/tests/extension/mockextensions/mock1.py @@ -1,5 +1,7 @@ """A mock extension named `mock1` for testing purposes.""" + # by the test functions. +import asyncio def _jupyter_server_extension_paths(): @@ -9,3 +11,8 @@ def _jupyter_server_extension_paths(): def _load_jupyter_server_extension(serverapp): serverapp.mockI = True serverapp.mock_shared = "I" + + +async def _start_jupyter_server_extension(serverapp): + await asyncio.sleep(0.1) + serverapp.mock1_started = True diff --git a/tests/extension/test_app.py b/tests/extension/test_app.py index 7e1671869d..74fb343a09 100644 --- a/tests/extension/test_app.py +++ b/tests/extension/test_app.py @@ -142,6 +142,10 @@ async def test_load_parallel_extensions(monkeypatch, jp_environ): async def test_start_extension(jp_serverapp, mock_extension): await jp_serverapp._post_start() assert mock_extension.started + assert hasattr( + jp_serverapp, "mock1_started" + ), "Failed because the `_start_jupyter_server_extension` function in 'mock1.py' was never called" + assert jp_serverapp.mock1_started async def test_stop_extension(jp_serverapp, caplog): From 481914c70e0ff7eac5672a8572c0dcbb0e1bc52d Mon Sep 17 00:00:00 2001 From: Zach Sailer Date: Thu, 25 Apr 2024 12:38:19 -0700 Subject: [PATCH 3/8] Add a no-op for backwards compatibility --- jupyter_server/extension/manager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index f51be19ed7..488bdbd622 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -120,16 +120,20 @@ def _get_loader(self): return loader def _get_starter(self): - """Get a linker.""" + """Get a starter function.""" if self.app: linker = self.app._start_jupyter_server_extension else: + + async def _noop_start(serverapp): + return + linker = getattr( self.module, # Search for a _start_jupyter_extension "_start_jupyter_server_extension", - # Otherwise return a dummy function. - lambda serverapp: None, + # Otherwise return a no-op function. + _noop_start, ) return linker From ccf5dd2d0e2f5e2f12ba988e0e5f63dc09c81c5c Mon Sep 17 00:00:00 2001 From: Zach Sailer Date: Thu, 25 Apr 2024 12:48:29 -0700 Subject: [PATCH 4/8] Only log message about starting extensions in debug mode --- jupyter_server/extension/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter_server/extension/manager.py b/jupyter_server/extension/manager.py index 488bdbd622..80510e21a6 100644 --- a/jupyter_server/extension/manager.py +++ b/jupyter_server/extension/manager.py @@ -416,7 +416,7 @@ async def start_extension(self, name): "%s | extension failed starting with message: %r", name, e, exc_info=True ) else: - self.log.info("%s | extension was successfully started.", name) + self.log.debug("%s | extension was successfully started.", name) async def stop_extension(self, name, apps): """Call the shutdown hooks in the specified apps.""" From 1abbce9c07ee284d24cf09bcc5711e74c1ad78d1 Mon Sep 17 00:00:00 2001 From: Zach Sailer Date: Thu, 25 Apr 2024 13:05:08 -0700 Subject: [PATCH 5/8] Linting fix --- docs/source/developers/extensions.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/developers/extensions.rst b/docs/source/developers/extensions.rst index 363795faf4..7d1f24e431 100644 --- a/docs/source/developers/extensions.rst +++ b/docs/source/developers/extensions.rst @@ -356,7 +356,8 @@ Starting asynchronous tasks from an ExtensionApp .. versionadded:: 2.15.0 -An ``ExtensionApp`` can start asynchronous tasks after Jupyter Server's event loop is started by overriding its ``_start_jupyter_server_extension()`` method. This can be helpful for setting up e.g. background tasks. +An ``ExtensionApp`` can start asynchronous tasks after Jupyter Server's event loop is started by overriding its +``_start_jupyter_server_extension()`` method. This can be helpful for setting up e.g. background tasks. Here is a basic (pseudo) code example: From 2f393845e9d6680fd78cbdee42f1fb302a9b24d5 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:49:12 +0000 Subject: [PATCH 6/8] Debug --- jupyter_server/serverapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 8e19658fc5..967bc4fcd9 100644 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -3124,7 +3124,9 @@ async def _post_start(self): """ try: await self.extension_manager.start_all_extensions() + print("OKOKOK") except Exception as err: + print("NOPE", err) self.log.error(err) info = self.log.info From 41bd1890466a4a4f0938020c23419fd9ac090262 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:57:34 +0000 Subject: [PATCH 7/8] Info --- jupyter_server/serverapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 967bc4fcd9..9d5f106b8e 100644 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -3124,9 +3124,9 @@ async def _post_start(self): """ try: await self.extension_manager.start_all_extensions() - print("OKOKOK") + self.log.info("OKOKOK") except Exception as err: - print("NOPE", err) + self.log.info(f"NOPE {err}") self.log.error(err) info = self.log.info From bc033df84871f9e297ed935fc4b356eb0d4dc9f1 Mon Sep 17 00:00:00 2001 From: krassowski <5832902+krassowski@users.noreply.github.com> Date: Tue, 26 Nov 2024 22:24:28 +0000 Subject: [PATCH 8/8] Early return? --- jupyter_server/serverapp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jupyter_server/serverapp.py b/jupyter_server/serverapp.py index 9d5f106b8e..2812185376 100644 --- a/jupyter_server/serverapp.py +++ b/jupyter_server/serverapp.py @@ -3129,6 +3129,8 @@ async def _post_start(self): self.log.info(f"NOPE {err}") self.log.error(err) + self.log.info("Returning early without criticals") + return info = self.log.info for line in self.running_server_info(kernel_count=False).split("\n"): info(line)