diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27b46bf2..c55b7c48 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,8 +18,8 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.13' cache: 'pip' @@ -38,7 +38,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13' ] + python-version: [ '3.10', '3.11', '3.12', '3.13' ] steps: - name: Checkout @@ -49,6 +49,8 @@ jobs: python-version: ${{ matrix.python-version }} cache: 'pip' + - uses: mamba-org/setup-micromamba@v2 + - name: Install hatch run: | python3 -m pip install --upgrade pip diff --git a/jupyverse_api/jupyverse_api/asgi_websocket_transport.py b/jupyverse_api/jupyverse_api/asgi_websocket_transport.py index 5b0b4f0f..819b75c7 100644 --- a/jupyverse_api/jupyverse_api/asgi_websocket_transport.py +++ b/jupyverse_api/jupyverse_api/asgi_websocket_transport.py @@ -32,9 +32,7 @@ def __init__(self, event: wsproto.events.Event) -> None: class ASGIWebSocketAsyncNetworkStream(AsyncNetworkStream): - def __init__( - self, app: ASGIApp, scope: Scope, task_group: anyio.abc.TaskGroup - ) -> None: + def __init__(self, app: ASGIApp, scope: Scope, task_group: anyio.abc.TaskGroup) -> None: self.app = app self.scope = scope self._task_group = task_group @@ -71,9 +69,7 @@ async def __aenter__( async def __aexit__(self, exc_type, exc_val, exc_tb) -> typing.Union[bool, None]: return await self._exit_stack.__aexit__(exc_type, exc_val, exc_tb) - async def read( - self, max_bytes: int, timeout: typing.Optional[float] = None - ) -> bytes: + async def read(self, max_bytes: int, timeout: typing.Optional[float] = None) -> bytes: message: Message = await self.receive(timeout=timeout) type = message["type"] @@ -93,9 +89,7 @@ async def read( return self.connection.send(event) - async def write( - self, buffer: bytes, timeout: typing.Optional[float] = None - ) -> None: + async def write(self, buffer: bytes, timeout: typing.Optional[float] = None) -> None: self.connection.receive_data(buffer) for event in self.connection.events(): if isinstance(event, wsproto.events.Request): @@ -169,9 +163,7 @@ def __init__(self, *args, **kwargs) -> None: async def __aenter__(self) -> "ASGIWebSocketTransport": async with contextlib.AsyncExitStack() as stack: - self._task_group = await stack.enter_async_context( - anyio.create_task_group() - ) + self._task_group = await stack.enter_async_context(anyio.create_task_group()) self.exit_stack = stack.pop_all() return self @@ -191,9 +183,7 @@ async def handle_async_request(self, request: Request) -> Response: if scheme in {"ws", "wss"} or headers.get("upgrade") == "websocket": subprotocols: list[str] = [] - if ( - subprotocols_header := headers.get("sec-websocket-protocol") - ) is not None: + if (subprotocols_header := headers.get("sec-websocket-protocol")) is not None: subprotocols = subprotocols_header.split(",") scope = { diff --git a/jupyverse_api/jupyverse_api/cli.py b/jupyverse_api/jupyverse_api/cli.py index 3d6c0f41..2013320b 100644 --- a/jupyverse_api/jupyverse_api/cli.py +++ b/jupyverse_api/jupyverse_api/cli.py @@ -84,6 +84,18 @@ type=str, help="Disable plugin.", ) +@click.option( + "--timeout", + type=float, + default=None, + help="The timeout for starting Jupyverse.", +) +@click.option( + "--stop-timeout", + type=float, + default=1, + help="The timeout for stopping Jupyverse.", +) def main( debug: bool = False, show_config: bool = False, @@ -96,6 +108,8 @@ def main( disable: tuple[str, ...] = (), allow_origin: tuple[str, ...] = (), query_param: tuple[str, ...] = (), + timeout: float | None = None, + stop_timeout: float = 1, ) -> None: query_params_dict = {} for qp in query_param: @@ -118,6 +132,8 @@ def main( show_config=show_config, help_all=help_all, backend=backend, + timeout=timeout, + stop_timeout=stop_timeout, ) # type: ignore diff --git a/jupyverse_api/jupyverse_api/environments/__init__.py b/jupyverse_api/jupyverse_api/environments/__init__.py new file mode 100644 index 00000000..c2f8dd90 --- /dev/null +++ b/jupyverse_api/jupyverse_api/environments/__init__.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod + +from fastapi import APIRouter, Depends + +from jupyverse_api import Router + +from ..app import App +from ..auth import Auth, User +from .models import CreateEnvironment, Environment, EnvironmentStatus + + +class Environments(Router, ABC): + def __init__(self, app: App, auth: Auth): + super().__init__(app=app) + + router = APIRouter() + + @router.delete("/api/environments/{environment_id}", status_code=204) + async def delete_environment( + environment_id: str, + user: User = Depends(auth.current_user(permissions={"sessions": ["write"]})), + ): + return await self.delete_environment(environment_id, user) + + @router.post( + "/api/environments", + status_code=201, + response_model=Environment, + ) + async def create_environment( + environment: CreateEnvironment, + user: User = Depends(auth.current_user(permissions={"sessions": ["write"]})), + ) -> Environment: + return await self.create_environment(environment, user) + + @router.get("/api/environments/wait/{environment_id}") + async def wait_for_environment(environment_id: str) -> None: + return await self.wait_for_environment(environment_id) + + @router.get("/api/environments/status/{environment_id}") + async def get_status(id: str) -> EnvironmentStatus: + return await self.get_status(id) + + self.include_router(router) + + @abstractmethod + async def delete_environment( + self, + id: str, + user: User, + ) -> None: ... + + @abstractmethod + async def create_environment( + self, + environment: CreateEnvironment, + user: User, + ) -> Environment: ... + + @abstractmethod + async def wait_for_environment(self, id: str) -> None: ... + + @abstractmethod + async def get_status(self, id: str) -> EnvironmentStatus: ... + + @abstractmethod + async def run_in_environment(self, id: str, command: str) -> int: ... + + @abstractmethod + def add_package_manager(self, name: str, package_manager: PackageManager): ... + + +class PackageManager(ABC): + @abstractmethod + async def create_environment(self, environment_file_path: str) -> str: ... + + @abstractmethod + async def delete_environment(self, id: str) -> None: ... + + @abstractmethod + async def wait_for_environment(self, id: str) -> None: ... + + @abstractmethod + async def get_status(self, id: str) -> EnvironmentStatus: ... + + @abstractmethod + async def run_in_environment(self, id: str, command: str) -> int: ... diff --git a/jupyverse_api/jupyverse_api/environments/models.py b/jupyverse_api/jupyverse_api/environments/models.py new file mode 100644 index 00000000..2c5be856 --- /dev/null +++ b/jupyverse_api/jupyverse_api/environments/models.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel + +EnvironmentStatus = ( + Literal["package manager not found"] + | Literal["environment uninitialized"] + | Literal["environment creation start"] + | Literal["environment creation success"] + | Literal["environment creation error"] + | Literal["environment file not found"] + | Literal["environment file not readable"] +) + + +class CreateEnvironment(BaseModel): + package_manager_name: str + environment_file_path: str + + +class Environment(BaseModel): + id: str + status: EnvironmentStatus diff --git a/jupyverse_api/jupyverse_api/kernel/__init__.py b/jupyverse_api/jupyverse_api/kernel/__init__.py index b1e93a12..574a9670 100644 --- a/jupyverse_api/jupyverse_api/kernel/__init__.py +++ b/jupyverse_api/jupyverse_api/kernel/__init__.py @@ -46,7 +46,11 @@ def __init__(self) -> None: ) @abstractmethod - async def start(self, *, task_status: TaskStatus[None] = TASK_STATUS_IGNORED,) -> None: ... + async def start( + self, + *, + task_status: TaskStatus[None] = TASK_STATUS_IGNORED, + ) -> None: ... @abstractmethod async def stop(self) -> None: ... diff --git a/jupyverse_api/jupyverse_api/kernels/__init__.py b/jupyverse_api/jupyverse_api/kernels/__init__.py index 879d0963..0212d1f6 100644 --- a/jupyverse_api/jupyverse_api/kernels/__init__.py +++ b/jupyverse_api/jupyverse_api/kernels/__init__.py @@ -251,4 +251,3 @@ class KernelsConfig(Config): default=None, ) require_yjs: bool = False - kernelenv_path: str = "" diff --git a/jupyverse_api/jupyverse_api/kernels/models.py b/jupyverse_api/jupyverse_api/kernels/models.py index 19397af1..b96da262 100644 --- a/jupyverse_api/jupyverse_api/kernels/models.py +++ b/jupyverse_api/jupyverse_api/kernels/models.py @@ -6,6 +6,7 @@ class KernelInfo(BaseModel): name: str | None = None id: str | None = None + environment_id: str | None = None class CreateSession(BaseModel): diff --git a/jupyverse_api/pyproject.toml b/jupyverse_api/pyproject.toml index 28fbc773..4f54ae82 100644 --- a/jupyverse_api/pyproject.toml +++ b/jupyverse_api/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "importlib_metadata >=3.6; python_version<'3.10'", "pydantic >=2,<3", "fastapi >=0.95.0,<1", - "fps >=0.5.1,<0.6.0", + "fps >=0.5.2,<0.6.0", "anyio >=3.6.2,<5", "anyioutils >=0.7.4", ] diff --git a/plugins/environment_micromamba/COPYING.md b/plugins/environment_micromamba/COPYING.md new file mode 100644 index 00000000..a9eec5ed --- /dev/null +++ b/plugins/environment_micromamba/COPYING.md @@ -0,0 +1,59 @@ +# Licensing terms + +This project is licensed under the terms of the Modified BSD License +(also known as New or Revised or 3-Clause BSD), as follows: + +- Copyright (c) 2025-, Jupyter Development Team + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the Jupyter Development Team nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +## About the Jupyter Development Team + +The Jupyter Development Team is the set of all contributors to the Jupyter project. +This includes all of the Jupyter subprojects. + +The core team that coordinates development on GitHub can be found here: +https://github.com/jupyter/. + +## Our Copyright Policy + +Jupyter uses a shared copyright model. Each contributor maintains copyright +over their contributions to Jupyter. But, it is important to note that these +contributions are typically only changes to the repositories. Thus, the Jupyter +source code, in its entirety is not the copyright of any single person or +institution. Instead, it is the collective copyright of the entire Jupyter +Development Team. If individual contributors want to maintain a record of what +changes/contributions they have specific copyright on, they should indicate +their copyright in the commit message of the change, when they commit the +change to one of the Jupyter repositories. + +With this in mind, the following banner should be used in any source code file +to indicate the copyright and license terms: + + # Copyright (c) Jupyter Development Team. + # Distributed under the terms of the Modified BSD License. diff --git a/plugins/environment_micromamba/README.md b/plugins/environment_micromamba/README.md new file mode 100644 index 00000000..4d6ec4d9 --- /dev/null +++ b/plugins/environment_micromamba/README.md @@ -0,0 +1,3 @@ +# fps-environment-micromamba + +An FPS plugin for micromamba environments. diff --git a/plugins/environment_micromamba/fps_environment_micromamba/__init__.py b/plugins/environment_micromamba/fps_environment_micromamba/__init__.py new file mode 100644 index 00000000..c1efa4a3 --- /dev/null +++ b/plugins/environment_micromamba/fps_environment_micromamba/__init__.py @@ -0,0 +1,6 @@ +import importlib.metadata + +try: + __version__ = importlib.metadata.version("fps_environment_micromamba") +except importlib.metadata.PackageNotFoundError: + __version__ = "unknown" diff --git a/plugins/environment_micromamba/fps_environment_micromamba/main.py b/plugins/environment_micromamba/fps_environment_micromamba/main.py new file mode 100644 index 00000000..7f4ad234 --- /dev/null +++ b/plugins/environment_micromamba/fps_environment_micromamba/main.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from anyio import Event, create_task_group +from fps import Module + +from jupyverse_api.environments import Environments + +from .micromamba import Micromamba + + +class EnvironmentMicromambaModule(Module): + async def prepare(self) -> None: + self.stop_event = Event() + environments = await self.get(Environments) # type: ignore[type-abstract] + async with create_task_group() as tg: + environments.add_package_manager("micromamba", Micromamba(tg)) + self.done() + await self.stop_event.wait() + + async def stop(self) -> None: + self.stop_event.set() diff --git a/plugins/environment_micromamba/fps_environment_micromamba/micromamba.py b/plugins/environment_micromamba/fps_environment_micromamba/micromamba.py new file mode 100644 index 00000000..37a2e213 --- /dev/null +++ b/plugins/environment_micromamba/fps_environment_micromamba/micromamba.py @@ -0,0 +1,87 @@ +from typing import TypedDict +from uuid import uuid4 + +import anyio +import yaml # type: ignore[import-untyped] +from anyio import Event, open_process, run_process +from anyio.abc import TaskGroup +from anyio.streams.text import TextReceiveStream + +from jupyverse_api.environments import EnvironmentStatus, PackageManager + + +class EnvironmentType(TypedDict): + name: str + status: EnvironmentStatus + done: Event + + +class Micromamba(PackageManager): + def __init__(self, task_group: TaskGroup) -> None: + self._task_group = task_group + self._environments: dict[str, EnvironmentType] = {} + + async def delete_environment(self, id: str) -> None: + name = self._environments[id]["name"] + cmd = f"micromamba env remove -n {name} --yes" + await run_process(cmd) + + async def create_environment(self, environment_file_path: str) -> str: + _id = uuid4().hex + environment: EnvironmentType = { + "name": "", + "status": "environment uninitialized", + "done": Event(), + } + self._environments[_id] = environment + path = anyio.Path(environment_file_path) + if not await path.is_file(): + environment["status"] = "environment file not found" + environment["done"].set() + return _id + try: + env = await path.read_text() + environment["name"] = yaml.load(env, Loader=yaml.CLoader)["name"] + except BaseException: + environment["status"] = "environment file not readable" + environment["done"].set() + return _id + environment["status"] = "environment creation start" + cmd = "micromamba --help" + try: + await run_process(cmd) + except BaseException: + environment["status"] = "package manager not found" + environment["done"].set() + return _id + cmd = f"micromamba create -f {environment_file_path} --yes" + self._task_group.start_soon(self._create_environment, environment, cmd) + return _id + + async def _create_environment(self, environment, cmd) -> None: + try: + await run_process(cmd) + except BaseException: + environment["status"] = "environment creation error" + else: + environment["status"] = "environment creation success" + environment["done"].set() + + async def wait_for_environment(self, id: str) -> None: + await self._environments[id]["done"].wait() + + async def get_status(self, id: str) -> EnvironmentStatus: + return self._environments[id]["status"] + + async def run_in_environment(self, id: str, command: str) -> int: + name = self._environments[id]["name"] + cmd = ( + """bash -c 'eval "$(micromamba shell hook --shell bash)";""" + + f"micromamba activate {name}; {command}" + + "' & echo $!" + ) + process = await open_process(cmd) + assert process.stdout is not None + async for text in TextReceiveStream(process.stdout): + break + return int(text) diff --git a/plugins/environment_micromamba/fps_environment_micromamba/py.typed b/plugins/environment_micromamba/fps_environment_micromamba/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/plugins/environment_micromamba/pyproject.toml b/plugins/environment_micromamba/pyproject.toml new file mode 100644 index 00000000..27b45805 --- /dev/null +++ b/plugins/environment_micromamba/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "fps_environment_micromamba" +version = "0.1.0" +description = "An FPS plugin for micromamba environments" +keywords = ["jupyter", "server", "fastapi", "plugins"] +requires-python = ">=3.9" +dependencies = [ + "jupyverse-api >=0.12.0,<0.13.0", + "pyyaml >=6.0.2,<7.0.0", +] + +[[project.authors]] +name = "Jupyter Development Team" +email = "jupyter@googlegroups.com" + +[project.readme] +file = "README.md" +content-type = "text/markdown" + +[project.license] +text = "BSD 3-Clause License" + +[project.urls] +Homepage = "https://jupyter.org" + +[project.entry-points] +"fps.modules" = {environment_micromamba = "fps_environment_micromamba.main:EnvironmentMicromambaModule"} +"jupyverse.modules" = {environment_micromamba = "fps_environment_micromamba.main:EnvironmentMicromambaModule"} diff --git a/plugins/environments/COPYING.md b/plugins/environments/COPYING.md new file mode 100644 index 00000000..a9eec5ed --- /dev/null +++ b/plugins/environments/COPYING.md @@ -0,0 +1,59 @@ +# Licensing terms + +This project is licensed under the terms of the Modified BSD License +(also known as New or Revised or 3-Clause BSD), as follows: + +- Copyright (c) 2025-, Jupyter Development Team + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +Neither the name of the Jupyter Development Team nor the names of its +contributors may be used to endorse or promote products derived from this +software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +## About the Jupyter Development Team + +The Jupyter Development Team is the set of all contributors to the Jupyter project. +This includes all of the Jupyter subprojects. + +The core team that coordinates development on GitHub can be found here: +https://github.com/jupyter/. + +## Our Copyright Policy + +Jupyter uses a shared copyright model. Each contributor maintains copyright +over their contributions to Jupyter. But, it is important to note that these +contributions are typically only changes to the repositories. Thus, the Jupyter +source code, in its entirety is not the copyright of any single person or +institution. Instead, it is the collective copyright of the entire Jupyter +Development Team. If individual contributors want to maintain a record of what +changes/contributions they have specific copyright on, they should indicate +their copyright in the commit message of the change, when they commit the +change to one of the Jupyter repositories. + +With this in mind, the following banner should be used in any source code file +to indicate the copyright and license terms: + + # Copyright (c) Jupyter Development Team. + # Distributed under the terms of the Modified BSD License. diff --git a/plugins/environments/README.md b/plugins/environments/README.md new file mode 100644 index 00000000..20271c1b --- /dev/null +++ b/plugins/environments/README.md @@ -0,0 +1,3 @@ +# fps-environments + +An FPS plugin for the environments API. diff --git a/plugins/environments/fps_environments/__init__.py b/plugins/environments/fps_environments/__init__.py new file mode 100644 index 00000000..c5a67e76 --- /dev/null +++ b/plugins/environments/fps_environments/__init__.py @@ -0,0 +1,6 @@ +import importlib.metadata + +try: + __version__ = importlib.metadata.version("fps_environments") +except importlib.metadata.PackageNotFoundError: + __version__ = "unknown" diff --git a/plugins/environments/fps_environments/environments.py b/plugins/environments/fps_environments/environments.py new file mode 100644 index 00000000..3f352aef --- /dev/null +++ b/plugins/environments/fps_environments/environments.py @@ -0,0 +1,67 @@ +import structlog + +from jupyverse_api.app import App +from jupyverse_api.auth import Auth, User +from jupyverse_api.environments import ( + CreateEnvironment, + Environment, + Environments, + EnvironmentStatus, + PackageManager, +) + +logger = structlog.get_logger() + + +class _Environments(Environments): + def __init__( + self, + app: App, + auth: Auth, + ) -> None: + super().__init__(app=app, auth=auth) + self._package_managers: dict[str, PackageManager] = {} + self._pm_for_id: dict[str, PackageManager] = {} + + async def delete_environment( + self, + id: str, + user: User, + ) -> None: + package_manager = self._pm_for_id[id] + await package_manager.delete_environment(id) + + async def create_environment( + self, + environment: CreateEnvironment, + user: User, + ) -> Environment: + package_manager_name = environment.package_manager_name + environment_file_path = environment.environment_file_path + if package_manager_name in self._package_managers: + logger.info( + "Creating environment", + package_manager=package_manager_name, + environment_file=environment_file_path, + ) + package_manager = self._package_managers[package_manager_name] + _id = await package_manager.create_environment(environment_file_path) + self._pm_for_id[_id] = package_manager + status = await package_manager.get_status(_id) + return Environment(id=_id, status=status) + raise RuntimeError(f"Package manager not found: {package_manager_name}") + + async def wait_for_environment(self, id: str) -> None: + package_manager = self._pm_for_id[id] + await package_manager.wait_for_environment(id) + + async def get_status(self, id: str) -> EnvironmentStatus: + package_manager = self._pm_for_id[id] + return await package_manager.get_status(id) + + async def run_in_environment(self, id: str, command: str) -> int: + package_manager = self._pm_for_id[id] + return await package_manager.run_in_environment(id, command) + + def add_package_manager(self, name: str, package_manager: PackageManager): + self._package_managers[name] = package_manager diff --git a/plugins/environments/fps_environments/main.py b/plugins/environments/fps_environments/main.py new file mode 100644 index 00000000..5c27531c --- /dev/null +++ b/plugins/environments/fps_environments/main.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from fps import Module + +from jupyverse_api.app import App +from jupyverse_api.auth import Auth +from jupyverse_api.environments import Environments + +from .environments import _Environments + + +class EnvironmentsModule(Module): + async def prepare(self) -> None: + app = await self.get(App) + auth = await self.get(Auth) # type: ignore[type-abstract] + environments = _Environments(app, auth) + self.put(environments, Environments) diff --git a/plugins/environments/fps_environments/py.typed b/plugins/environments/fps_environments/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/plugins/environments/pyproject.toml b/plugins/environments/pyproject.toml new file mode 100644 index 00000000..784d2280 --- /dev/null +++ b/plugins/environments/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "fps_environments" +version = "0.1.0" +description = "An FPS plugin for the environments API" +keywords = ["jupyter", "server", "fastapi", "plugins"] +requires-python = ">=3.9" +dependencies = [ + "jupyverse-api >=0.12.0,<0.13.0", +] + +[[project.authors]] +name = "Jupyter Development Team" +email = "jupyter@googlegroups.com" + +[project.readme] +file = "README.md" +content-type = "text/markdown" + +[project.license] +text = "BSD 3-Clause License" + +[project.urls] +Homepage = "https://jupyter.org" + +[project.entry-points] +"fps.modules" = {environments = "fps_environments.main:EnvironmentsModule"} +"jupyverse.modules" = {environments = "fps_environments.main:EnvironmentsModule"} diff --git a/plugins/environments/tests/test_environments.py b/plugins/environments/tests/test_environments.py new file mode 100644 index 00000000..f2bef5bf --- /dev/null +++ b/plugins/environments/tests/test_environments.py @@ -0,0 +1,102 @@ +import json +import platform + +import httpx +import pytest +from anyio import fail_after, sleep +from fps import get_root_module + +from jupyverse_api.environments import Environments + +ENVIRONMENT = """\ +name: my-test-env +dependencies: + - numpy + - ipykernel +""" + + +@pytest.mark.anyio +@pytest.mark.skipif(platform.system() == "Windows", reason="Doesn't support Windows") +async def test_kernel_environment_micromamba(tmp_path): + config = { + "jupyverse": { + "type": "jupyverse", + "config": { + "start_server": False, + }, + "modules": { + "app": { + "type": "app", + }, + "auth": { + "type": "noauth", + }, + "environment_micromamba": { + "type": "environment_micromamba", + }, + "environments": { + "type": "environments", + }, + "kernel_subprocess": { + "type": "kernel_subprocess", + }, + "frontend": { + "type": "frontend", + }, + "kernels": { + "type": "kernels", + }, + "file_watcher": { + "type": "file_watcher", + }, + }, + } + } + + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module as root_module: + environments = await root_module.get(Environments) + app = root_module.app + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client: + environment_file_path = tmp_path / "environment.yml" + environment_file_path.write_text(ENVIRONMENT) + data = { + "package_manager_name": "micromamba", + "environment_file_path": str(environment_file_path), + } + r = await client.post("/api/environments", json=data) + assert r.status_code == 201 + response = r.json() + assert response["status"] == "environment creation start" + environment_id = response["id"] + r = await client.get(f"/api/environments/wait/{environment_id}") + assert r.status_code == 200 + data = { + "name": "Untitled.ipynb", + "path": "012-abc", + "type": "notebook", + "kernel": { + "name": "python3", + "environment_id": environment_id, + }, + } + r = await client.post("/api/sessions", content=json.dumps(data)) + assert r.status_code == 201 + test_file_path = tmp_path / "foo.txt" + code_file_path = tmp_path / "foo.py" + code_file_path.write_text("import numpy; print(numpy)") + cmd = f"python {code_file_path} > {test_file_path}" + await environments.run_in_environment(environment_id, cmd) + with fail_after(1): + while True: + await sleep(0.1) + if test_file_path.is_file(): + content = test_file_path.read_text() + if content: + assert content.startswith(" AsyncGenerator[set[FileChange], None]: async for changes in awatch(path, stop_event=stop_event): diff --git a/plugins/kernel_subprocess/fps_kernel_subprocess/kernel_subprocess.py b/plugins/kernel_subprocess/fps_kernel_subprocess/kernel_subprocess.py index 042f0dbc..cef81148 100644 --- a/plugins/kernel_subprocess/fps_kernel_subprocess/kernel_subprocess.py +++ b/plugins/kernel_subprocess/fps_kernel_subprocess/kernel_subprocess.py @@ -10,10 +10,10 @@ from typing import cast import anyio -from anyio import TASK_STATUS_IGNORED, create_task_group, open_file, open_process, run_process +from anyio import TASK_STATUS_IGNORED, create_task_group, open_file, open_process, to_thread from anyio.abc import TaskStatus -from anyio.streams.text import TextReceiveStream +from jupyverse_api.environments import Environments from jupyverse_api.kernel import Kernel from .connect import ( @@ -33,8 +33,9 @@ class KernelSubprocess(Kernel): connection_file: str kernel_cwd: str | None capture_output: bool + environments: Environments connection_cfg: cfg_t | None = None - kernelenv_path: str = "" + environment_id: str = "" def __post_init__(self): super().__init__() @@ -82,25 +83,9 @@ async def start(self, *, task_status: TaskStatus[None] = TASK_STATUS_IGNORED) -> stdout = None stderr = None kernel_cwd = self.kernel_cwd if self.kernel_cwd else None - kernelenv = "" - if self.kernelenv_path: - path = anyio.Path(self.kernelenv_path) - if await path.is_file(): - kernelenv = await path.read_text() - if kernelenv: - import yaml # type: ignore[import-untyped] - env_name = yaml.load(kernelenv, Loader=yaml.CLoader)["name"] - cmd = f"micromamba create -f {self.kernelenv_path} --yes" - result = await run_process(cmd) - if result.returncode == 0: - cmd = """bash -c 'eval "$(micromamba shell hook --shell bash)";""" + \ - f"micromamba activate {env_name};" + \ - " ".join(launch_kernel_cmd) + "' & echo $!" - process = await open_process(cmd) - assert process.stdout is not None - async for text in TextReceiveStream(process.stdout): - self._pid = int(text) - break + if self.environment_id: + cmd = " ".join(launch_kernel_cmd) + self._pid = await self.environments.run_in_environment(self.environment_id, cmd) else: if launch_kernel_cmd and launch_kernel_cmd[0] in { "python", @@ -148,7 +133,14 @@ async def stop(self) -> None: path = anyio.Path(self.connection_file) await path.unlink(missing_ok=True) else: - os.kill(self._pid, signal.SIGTERM) + try: + os.kill(self._pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + await to_thread.run_sync(os.waitpid, self._pid, 0) + except ChildProcessError: + pass await self.shell_channel.stop() await self.stdin_channel.stop() diff --git a/plugins/kernel_subprocess/fps_kernel_subprocess/main.py b/plugins/kernel_subprocess/fps_kernel_subprocess/main.py index 48c830e6..cee77c1d 100644 --- a/plugins/kernel_subprocess/fps_kernel_subprocess/main.py +++ b/plugins/kernel_subprocess/fps_kernel_subprocess/main.py @@ -2,6 +2,7 @@ from fps import Module +from jupyverse_api.environments import Environments from jupyverse_api.kernel import DefaultKernelFactory from .kernel_subprocess import KernelSubprocess @@ -9,5 +10,11 @@ class KernelSubprocessModule(Module): async def prepare(self) -> None: - default_kernel_factory = DefaultKernelFactory(KernelSubprocess) + environments = await self.get(Environments) # type: ignore[type-abstract] + + class _KernelSubprocess(KernelSubprocess): + def __init__(self, *args, **kwargs): + super().__init__(*args, environments=environments, **kwargs) + + default_kernel_factory = DefaultKernelFactory(_KernelSubprocess) self.put(default_kernel_factory) diff --git a/plugins/kernel_web_worker/fps_kernel_web_worker/kernel_web_worker.py b/plugins/kernel_web_worker/fps_kernel_web_worker/kernel_web_worker.py index 365d9cd7..f2210b2e 100644 --- a/plugins/kernel_web_worker/fps_kernel_web_worker/kernel_web_worker.py +++ b/plugins/kernel_web_worker/fps_kernel_web_worker/kernel_web_worker.py @@ -49,8 +49,10 @@ def callback(msg): self.js_callable, self.js_py_object = pyjs.create_callable(callback) self.higher_order_function = pyjs.js.Function( - "callback", "action", "kernel_id", - "kernel_web_worker(action, kernel_id, 0, callback);" + "callback", + "action", + "kernel_id", + "kernel_web_worker(action, kernel_id, 0, callback);", ) self.higher_order_function(self.js_callable, "start", self.kernel_id) await kernel_ready.wait() diff --git a/plugins/kernels/fps_kernels/kernel_server/server.py b/plugins/kernels/fps_kernels/kernel_server/server.py index e096478b..eef276fe 100644 --- a/plugins/kernels/fps_kernels/kernel_server/server.py +++ b/plugins/kernels/fps_kernels/kernel_server/server.py @@ -27,7 +27,7 @@ to_binary, ) -kernels: dict = {} +KERNELS: dict = {} class AcceptedWebSocket: @@ -52,7 +52,7 @@ def __init__( self, default_kernel_factory: DefaultKernelFactory, kernelspec_path: str = "", - kernelenv_path: str = "", + environment_id: str | None = None, kernel_cwd: str = "", connection_file: str = "", write_connection_file: bool = True, @@ -61,7 +61,7 @@ def __init__( self.default_kernel_factory = default_kernel_factory self.capture_kernel_output = capture_kernel_output self.kernelspec_path = kernelspec_path - self.kernelenv_path = kernelenv_path + self.environment_id = environment_id self.kernel_cwd = kernel_cwd self.connection_file = connection_file self.write_connection_file = write_connection_file @@ -111,7 +111,7 @@ async def start( self.kernel = self.default_kernel_factory( write_connection_file=self.write_connection_file, kernelspec_path=self.kernelspec_path, - kernelenv_path=self.kernelenv_path, + environment_id=self.environment_id, connection_file=self.connection_file, kernel_cwd=self.kernel_cwd, capture_output=self.capture_kernel_output, @@ -119,7 +119,7 @@ async def start( else: self.kernel = kernel_factory( kernelspec_path=self.kernelspec_path, - kernelenv_path=self.kernelenv_path, + environment_id=self.environment_id, connection_file=self.connection_file, kernel_cwd=self.kernel_cwd, capture_output=self.capture_kernel_output, diff --git a/plugins/kernels/fps_kernels/routes.py b/plugins/kernels/fps_kernels/routes.py index 5e6b91a1..99be6b13 100644 --- a/plugins/kernels/fps_kernels/routes.py +++ b/plugins/kernels/fps_kernels/routes.py @@ -27,9 +27,9 @@ from .kernel_driver.kernelspec import find_kernelspec, kernelspec_dirs from .kernel_driver.paths import jupyter_runtime_dir from .kernel_server.server import ( + KERNELS, AcceptedWebSocket, KernelServer, - kernels, ) logger = structlog.get_logger() @@ -58,7 +58,7 @@ def __init__( self.kernelspecs: dict = {} self.kernel_id_to_connection_file: dict[str, str] = {} self.sessions: dict[str, Session] = {} - self.kernels = kernels + self.kernels = KERNELS self._app = app self.stop_event = Event() self._stop_lock = Lock() @@ -89,6 +89,7 @@ async def stop(self) -> None: tg.start_soon(kernel["server"].stop) if kernel["driver"] is not None: tg.start_soon(kernel["driver"].stop) + self.kernels.clear() self.stop_event.set() self.task_group.cancel_scope.cancel() @@ -103,12 +104,12 @@ async def get_status( started = self._app.started_time.isoformat().replace("+00:00", "Z") last_activity = self._app.last_activity.isoformat().replace("+00:00", "Z") connections = sum( - [kernel["server"].connections for kernel in kernels.values() if "server" in kernel] + [kernel["server"].connections for kernel in self.kernels.values() if "server" in kernel] ) return { "started": started, "last_activity": last_activity, - "kernels": len(kernels), + "kernels": len(self.kernels), "connections": connections, } @@ -149,7 +150,7 @@ async def get_kernels( user: User, ): results = [] - for kernel_id, kernel in kernels.items(): + for kernel_id, kernel in self.kernels.items(): if kernel["server"]: connections = kernel["server"].connections last_activity = kernel["server"].last_activity["date"] @@ -175,9 +176,9 @@ async def delete_session( user: User, ): kernel_id = self.sessions[session_id].kernel.id - kernel_server = kernels[kernel_id]["server"] + kernel_server = self.kernels[kernel_id]["server"] await kernel_server.stop() - del kernels[kernel_id] + del self.kernels[kernel_id] if kernel_id in self.kernel_id_to_connection_file: del self.kernel_id_to_connection_file[kernel_id] del self.sessions[session_id] @@ -200,8 +201,8 @@ async def get_sessions( ): for session in self.sessions.values(): kernel_id = session.kernel.id - if kernel_id in kernels: - kernel_server = kernels[kernel_id]["server"] + if kernel_id in self.kernels: + kernel_server = self.kernels[kernel_id]["server"] session.kernel.last_activity = kernel_server.last_activity["date"] session.kernel.execution_state = kernel_server.last_activity["execution_state"] return list(self.sessions.values()) @@ -214,6 +215,7 @@ async def create_session( create_session = CreateSession(**(await request.json())) kernel_id = create_session.kernel.id kernel_name = create_session.kernel.name + environment_id = create_session.kernel.environment_id if kernel_name is not None: # launch a new ("internal") kernel kernel_cwd = Path(create_session.path).parent @@ -223,30 +225,33 @@ async def create_session( kernel_cwd = kernel_cwd.parent kernel_server = KernelServer( kernelspec_path=Path(find_kernelspec(kernel_name)).as_posix(), - kernelenv_path=self.kernels_config.kernelenv_path, + environment_id=environment_id, kernel_cwd=str(kernel_cwd), default_kernel_factory=self.default_kernel_factory, ) kernel_id = str(uuid.uuid4()) - kernels[kernel_id] = {"name": kernel_name, "server": kernel_server, "driver": None} - logger.info("Starting kernel", kernel_id=kernel_id, kernel_name=kernel_name) + self.kernels[kernel_id] = {"name": kernel_name, "server": kernel_server, "driver": None} + kwargs = dict(kernel_id=kernel_id, kernel_name=kernel_name) + if environment_id: + kwargs["environment_id"] = environment_id + logger.info("Starting kernel", **kwargs) kernel_factory = self.kernel_factories.get(kernel_name) await self.task_group.start(partial(kernel_server.start, kernel_factory=kernel_factory)) elif kernel_id is not None: # external or already running kernel - if kernel_id not in kernels: + if kernel_id not in self.kernels: raise HTTPException(status_code=404, detail=f"Kernel ID not found: {kernel_id}") - kernel_name = kernels[kernel_id]["name"] - if kernels[kernel_id]["server"] is None: + kernel_name = self.kernels[kernel_id]["name"] + if self.kernels[kernel_id]["server"] is None: kernel_server = KernelServer( connection_file=self.kernel_id_to_connection_file[kernel_id], write_connection_file=False, default_kernel_factory=self.default_kernel_factory, ) - kernels[kernel_id]["server"] = kernel_server + self.kernels[kernel_id]["server"] = kernel_server await self.task_group.start(partial(kernel_server.start, launch_kernel=False)) else: - kernel_server = kernels[kernel_id]["server"] + kernel_server = self.kernels[kernel_id]["server"] else: return session_id = str(uuid.uuid4()) @@ -275,8 +280,8 @@ async def interrupt_kernel( kernel_id, user: User, ): - if kernel_id in kernels: - kernel = kernels[kernel_id] + if kernel_id in self.kernels: + kernel = self.kernels[kernel_id] await kernel["server"].interrupt() result = { "id": kernel_id, @@ -292,8 +297,8 @@ async def restart_kernel( kernel_id, user: User, ): - if kernel_id in kernels: - kernel = kernels[kernel_id] + if kernel_id in self.kernels: + kernel = self.kernels[kernel_id] await self.task_group.start(kernel["server"].restart) result = { "id": kernel_id, @@ -315,7 +320,7 @@ async def execute_cell( r = await request.json() execution = Execution(**r) - if kernel_id in kernels: + if kernel_id in self.kernels: ynotebook = self.yjs.get_document(execution.document_id) ycells = [ycell for ycell in ynotebook.ycells if ycell["id"] == execution.cell_id] if not ycells: @@ -324,7 +329,7 @@ async def execute_cell( ycell = ycells[0] del ycell["outputs"][:] - kernel = kernels[kernel_id] + kernel = self.kernels[kernel_id] if not kernel["driver"]: kernel["driver"] = driver = KernelDriver( default_kernel_factory=self.default_kernel_factory, @@ -343,8 +348,8 @@ async def get_kernel( kernel_id, user: User, ): - if kernel_id in kernels: - kernel = kernels[kernel_id] + if kernel_id in self.kernels: + kernel = self.kernels[kernel_id] result = { "id": kernel_id, "name": kernel["name"], @@ -360,9 +365,9 @@ async def shutdown_kernel( user: User, ): logger.info("Stopping kernel", kernel_id=kernel_id) - if kernel_id in kernels: - await kernels[kernel_id]["server"].stop() - del kernels[kernel_id] + if kernel_id in self.kernels: + await self.kernels[kernel_id]["server"].stop() + del self.kernels[kernel_id] for session_id in [k for k, v in self.sessions.items() if v.kernel.id == kernel_id]: del self.sessions[session_id] return Response(status_code=HTTPStatus.NO_CONTENT.value) @@ -383,8 +388,8 @@ async def kernel_channels( ) await websocket.accept(subprotocol=subprotocol) accepted_websocket = AcceptedWebSocket(websocket, subprotocol) - if kernel_id in kernels: - kernel_server = kernels[kernel_id]["server"] + if kernel_id in self.kernels: + kernel_server = self.kernels[kernel_id]["server"] if kernel_server is None: # this is an external kernel # kernel is already launched, just start a kernel server @@ -393,7 +398,7 @@ async def kernel_channels( write_connection_file=False, ) await self.task_group.start(partial(kernel_server.start, launch_kernel=False)) - kernels[kernel_id]["server"] = kernel_server + self.kernels[kernel_id]["server"] = kernel_server await kernel_server.serve(accepted_websocket, session_id, permissions) async def watch_connection_files(self, path: Path) -> None: @@ -430,7 +435,7 @@ async def process_connection_files(self, changes: set[tuple[Change, str]]): list(self.kernel_id_to_connection_file.values()).index(path) ] del self.kernel_id_to_connection_file[kernel_id] - del kernels[kernel_id] + del self.kernels[kernel_id] elif change == Change.added: try: data = json.loads(Path(path).read_text()) @@ -441,7 +446,7 @@ async def process_connection_files(self, changes: set[tuple[Change, str]]): # looks like a kernel connection file kernel_id = str(uuid.uuid4()) self.kernel_id_to_connection_file[kernel_id] = path - kernels[kernel_id] = { + self.kernels[kernel_id] = { "name": data["kernel_name"], "server": None, "driver": None, diff --git a/plugins/kernels/tests/test_kernel_launcher.py b/plugins/kernels/tests/test_kernel_launcher.py index 4180ce97..8e5ca4a0 100644 --- a/plugins/kernels/tests/test_kernel_launcher.py +++ b/plugins/kernels/tests/test_kernel_launcher.py @@ -6,7 +6,6 @@ @pytest.mark.anyio -@pytest.mark.parametrize("anyio_backend", ["asyncio"]) async def test_kernel_launcher(): config = { "jupyverse": { @@ -19,11 +18,10 @@ async def test_kernel_launcher(): "type": "app", }, "auth": { - "type": "auth", - "config": { - "test": True, - "mode": "noauth", - }, + "type": "noauth", + }, + "environments": { + "type": "environments", }, "kernel_subprocess": { "type": "kernel_subprocess", @@ -41,7 +39,9 @@ async def test_kernel_launcher(): } } - async with get_root_module(config) as root_module: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module as root_module: app = root_module.app transport = httpx.ASGITransport(app=app) async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client: diff --git a/plugins/webdav/tests/test_webdav.py b/plugins/webdav/tests/test_webdav.py index 7d0bea9e..cd11ec6f 100644 --- a/plugins/webdav/tests/test_webdav.py +++ b/plugins/webdav/tests/test_webdav.py @@ -42,7 +42,9 @@ async def test_webdav(unused_tcp_port): } }, ) - async with get_root_module(config): + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module: webdav = easywebdav.connect( "127.0.0.1", port=unused_tcp_port, path="webdav", username="foo", password="bar" ) diff --git a/plugins/yjs/tests/test_yjs.py b/plugins/yjs/tests/test_yjs.py index 0f6f4078..9a82d27f 100644 --- a/plugins/yjs/tests/test_yjs.py +++ b/plugins/yjs/tests/test_yjs.py @@ -14,19 +14,21 @@ @pytest.mark.anyio async def test_concurrent_disconnect(tmp_path, anyio_backend_name): if ( - anyio_backend_name == "trio" and - sys.version_info >= (3, 13) and - sys.version_info < (3, 14) and - sys.platform == "darwin" + anyio_backend_name == "trio" + and ( + (sys.version_info >= (3, 13) and sys.version_info < (3, 14)) or + (sys.version_info >= (3, 11) and sys.version_info < (3, 12)) + ) + and sys.platform == "darwin" ): pytest.skip("Timeout") config = { "jupyverse": { "type": "jupyverse", - "config": { - "start_server": False, - }, + "config": { + "start_server": False, + }, "modules": { "app": { "type": "app", @@ -57,7 +59,9 @@ async def test_concurrent_disconnect(tmp_path, anyio_backend_name): } with capture_logs() as cap_logs: - async with get_root_module(config) as root_module: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module as root_module: app = root_module.app transport = ASGIWebSocketTransport(app=app) async with AsyncClient(transport=transport, base_url="http://testserver") as client: diff --git a/pyproject.toml b/pyproject.toml index 1fc947db..f25f9d65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = [ "hatchling",] +requires = ["hatchling"] build-backend = "hatchling.build" [project] @@ -9,8 +9,10 @@ description = "A set of FPS plugins implementing a Jupyter server" keywords = ["jupyter", "server", "fastapi", "plugins"] requires-python = ">=3.9" dependencies = [ - "fps[click,fastapi,anycorn] >=0.5.1,<0.6.0", + "fps[click,fastapi,anycorn] >=0.5.2,<0.6.0", "fps-contents >=0.10.2,<0.11.0", + "fps-environments >=0.1.0,<0.2.0", + "fps-environment-micromamba >=0.1.0,<0.2.0", "fps-file-id >=0.3.3,<0.4.0", "fps-file-watcher >=0.1.2,<0.2.0", "fps-kernel-subprocess >=0.1.4,<0.2.0", @@ -61,7 +63,7 @@ test = [ "ypywidgets-textual >=0.5.0,<0.6.0", "trio", ] -docs = [ "mkdocs", "mkdocs-material" ] +docs = ["mkdocs", "mkdocs-material"] [tool.hatch.envs.dev] # TODO: if/when hatch gets support for defining editable dependencies, the @@ -70,6 +72,8 @@ docs = [ "mkdocs", "mkdocs-material" ] pre-install-commands = [ "pip install -e ./jupyverse_api", "pip install -e ./plugins/contents", + "pip install -e ./plugins/environments", + "pip install -e ./plugins/environment_micromamba", "pip install -e ./plugins/file_id", "pip install -e ./plugins/file_watcher", "pip install -e ./plugins/file_watcher_poll", @@ -89,7 +93,6 @@ pre-install-commands = [ "pip install -e ./plugins/auth_jupyterhub", "pip install -e ./plugins/jupyterlab", "pip install -e ./plugins/notebook", - "pip install pyyaml", ] features = ["test"] @@ -102,6 +105,8 @@ lint = [ typecheck = """mypy \ ./jupyverse_api \ ./plugins/contents \ +./plugins/environments \ +./plugins/environment_micromamba \ ./plugins/file_id \ ./plugins/file_watcher \ ./plugins/file_watcher_poll \ @@ -130,9 +135,6 @@ features = ["docs"] build = "mkdocs build --clean --strict" serve = "mkdocs serve --dev-addr localhost:8000" -[tool.check-manifest] -ignore = [ ".*",] - [tool.ruff.lint] select = [ # pycodestyle @@ -153,9 +155,6 @@ select = [ line-length = 100 exclude = ["binder"] -[tool.hatch.version] -path = "jupyverse/__init__.py" - [tool.uv.workspace] members = ["plugins/*", "jupyverse_api"] @@ -165,6 +164,8 @@ fps-auth = { workspace = true } fps-auth-fief = { workspace = true } fps-auth-jupyterhub = { workspace = true } fps-contents = { workspace = true } +fps-environments = { workspace = true } +fps-environment-micromamba = { workspace = true } fps-file-id = { workspace = true } fps-file-watcher = { workspace = true } fps-file-watcher-poll = { workspace = true } diff --git a/tests/conftest.py b/tests/conftest.py index 956d74fc..a160bc17 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,15 +32,26 @@ def start_jupyverse(auth_mode, clear_users, cwd, unused_tcp_port): os.chdir(cwd) command_list = [ "jupyverse", - "--disable", "noauth", - "--disable", "auth_fief", - "--disable", "auth_jupyterhub", - "--disable", "notebook", - "--disable", "file_watcher_poll", - "--set", f"auth.mode={auth_mode}", - "--set", f"auth.clear_users={str(clear_users).lower()}", - "--set", "kernels.require_yjs=true", - "--port", str(unused_tcp_port), + "--disable", + "noauth", + "--disable", + "auth_fief", + "--disable", + "auth_jupyterhub", + "--disable", + "notebook", + "--disable", + "file_watcher_poll", + "--set", + f"auth.mode={auth_mode}", + "--set", + f"auth.clear_users={str(clear_users).lower()}", + "--set", + "kernels.require_yjs=true", + "--port", + str(unused_tcp_port), + "--timeout", + "10", ] p = subprocess.Popen(command_list) url = f"http://127.0.0.1:{unused_tcp_port}" diff --git a/tests/test_app.py b/tests/test_app.py index 8df906ff..b463c06e 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -28,7 +28,9 @@ async def test_mount_path(mount_path, unused_tcp_port): } } - async with AsyncClient() as http, get_root_module(config) as jupyverse_module: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with AsyncClient() as http, root_module as jupyverse_module: app = await jupyverse_module.get(App) router = APIRouter() diff --git a/tests/test_auth.py b/tests/test_auth.py index fd980f99..423e4283 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -22,6 +22,9 @@ "contents": { "type": "contents", }, + "environments": { + "type": "environments", + }, "file_id": { "type": "file_id", }, @@ -54,7 +57,9 @@ @pytest.mark.anyio async def test_kernel_channels_unauthenticated(unused_tcp_port): config = merge_config(CONFIG, {"jupyverse": {"config": {"port": unused_tcp_port}}}) - async with get_root_module(config): + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module: with pytest.raises(WebSocketUpgradeError): async with aconnect_ws( f"http://127.0.0.1:{unused_tcp_port}/api/kernels/kernel_id_0/channels?session_id=session_id_0", @@ -65,7 +70,9 @@ async def test_kernel_channels_unauthenticated(unused_tcp_port): @pytest.mark.anyio async def test_kernel_channels_authenticated(unused_tcp_port): config = merge_config(CONFIG, {"jupyverse": {"config": {"port": unused_tcp_port}}}) - async with get_root_module(config), AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module, AsyncClient() as http: await authenticate_client(http, unused_tcp_port) async with aconnect_ws( f"http://127.0.0.1:{unused_tcp_port}/api/kernels/kernel_id_0/channels?session_id=session_id_0", @@ -92,7 +99,9 @@ async def test_root_auth(auth_mode, unused_tcp_port): } }, ) - async with get_root_module(config), AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module, AsyncClient() as http: response = await http.get(f"http://127.0.0.1:{unused_tcp_port}/") if auth_mode == "noauth": expected = 302 @@ -121,7 +130,9 @@ async def test_no_auth(auth_mode, unused_tcp_port): } }, ) - async with get_root_module(config), AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module, AsyncClient() as http: response = await http.get(f"http://127.0.0.1:{unused_tcp_port}/lab") assert response.status_code == 200 @@ -144,7 +155,9 @@ async def test_token_auth(auth_mode, unused_tcp_port): } }, ) - async with get_root_module(config) as jupyverse, AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module as jupyverse, AsyncClient() as http: auth_config = await jupyverse.get(AuthConfig) # no token provided, should not work @@ -180,7 +193,9 @@ async def test_permissions(auth_mode, permissions, unused_tcp_port): } }, ) - async with get_root_module(config), AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module, AsyncClient() as http: await authenticate_client(http, unused_tcp_port, permissions=permissions) response = await http.get(f"http://127.0.0.1:{unused_tcp_port}/auth/user/me") if "admin" in permissions.keys(): diff --git a/tests/test_contents.py b/tests/test_contents.py index b38dbbb0..55ba27f8 100644 --- a/tests/test_contents.py +++ b/tests/test_contents.py @@ -93,7 +93,9 @@ async def test_tree(auth_mode, tmp_path, unused_tcp_port): } }, ) - async with get_root_module(config), AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module, AsyncClient() as http: response = await http.get( f"http://127.0.0.1:{unused_tcp_port}/api/contents", params={"content": 1} ) diff --git a/tests/test_execute.py b/tests/test_execute.py index 89391ad4..ff052571 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -29,6 +29,9 @@ "contents": { "type": "contents", }, + "environments": { + "type": "environments", + }, "file_id": { "type": "file_id", }, @@ -109,7 +112,9 @@ async def test_execute(auth_mode, unused_tcp_port): } }, ) - async with get_root_module(config), AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module, AsyncClient() as http: ws_url = url.replace("http", "ws", 1) name = "notebook1.ipynb" path = (Path("tests") / "data" / name).as_posix() diff --git a/tests/test_kernels.py b/tests/test_kernels.py index 65040555..a0e3eb6a 100644 --- a/tests/test_kernels.py +++ b/tests/test_kernels.py @@ -5,7 +5,7 @@ import pytest from anyio import create_task_group, sleep, sleep_forever from fps import get_root_module, merge_config -from fps_kernels.kernel_server.server import KernelServer, kernels +from fps_kernels.kernel_server.server import KERNELS, KernelServer from httpx import AsyncClient from httpx_ws import aconnect_ws @@ -17,9 +17,9 @@ CONFIG = { "jupyverse": { "type": "jupyverse", - "config": { - "start_server": False, - }, + "config": { + "start_server": False, + }, "modules": { "app": { "type": "app", @@ -33,6 +33,9 @@ "contents": { "type": "contents", }, + "environments": { + "type": "environments", + }, "frontend": { "type": "frontend", }, @@ -90,7 +93,9 @@ async def test_kernel_messages(auth_mode, capfd): } }, ) - async with get_root_module(config) as root_module: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module as root_module: app = root_module.app transport = ASGIWebSocketTransport(app=app) async with AsyncClient(transport=transport, base_url="http://testserver") as client: @@ -101,7 +106,7 @@ async def test_kernel_messages(auth_mode, capfd): ) async with create_task_group() as tg: await tg.start(kernel_server.start) - kernels[kernel_id] = {"server": kernel_server, "driver": None} + KERNELS[kernel_id] = {"server": kernel_server, "driver": None} # block msg_type_0 kernel_server.block_messages("msg_type_0") diff --git a/tests/test_settings.py b/tests/test_settings.py index f9518ea2..c119c7ac 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -22,6 +22,9 @@ "contents": { "type": "contents", }, + "environments": { + "type": "environments", + }, "file_id": { "type": "file_id", }, @@ -69,7 +72,9 @@ async def test_settings(auth_mode, unused_tcp_port): } }, ) - async with get_root_module(config), AsyncClient() as http: + root_module = get_root_module(config) + root_module._global_start_timeout = 10 + async with root_module, AsyncClient() as http: # get previous theme response = await http.get( f"http://127.0.0.1:{unused_tcp_port}/lab/api/settings/@jupyterlab/apputils-extension:themes"