Merge branch 'main' into cors_files

Author: gogasca

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.28.4
+    rev: 0.28.6
     hooks:
       - id: check-github-workflows
 
@@ -52,7 +52,7 @@ repos:
       - id: rst-inline-touching-normal
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.0"
+    rev: "v1.10.1"
     hooks:
       - id: mypy
         files: jupyter_server
@@ -61,7 +61,7 @@ repos:
           ["traitlets>=5.13", "jupyter_core>=5.5", "jupyter_client>=8.5"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.7
+    rev: v0.5.0
     hooks:
       - id: ruff
         types_or: [python, jupyter]

CHANGELOG.md

@@ -4,6 +4,34 @@ All notable changes to this project will be documented in this file.
 
 <!-- <START NEW CHANGELOG ENTRY> -->
 
+## 2.14.2
+
+([Full Changelog](https://github.com/jupyter-server/jupyter_server/compare/v2.14.1...b961d4eb499071c0c60e24f429c20d1e6a908a32))
+
+### Bugs fixed
+
+- Pass session_id during Websocket connect [#1440](https://github.com/jupyter-server/jupyter_server/pull/1440) ([@gogasca](https://github.com/gogasca))
+- Do not log environment variables passed to kernels [#1437](https://github.com/jupyter-server/jupyter_server/pull/1437) ([@krassowski](https://github.com/krassowski))
+
+### Maintenance and upkeep improvements
+
+- chore: update pre-commit hooks [#1441](https://github.com/jupyter-server/jupyter_server/pull/1441) ([@pre-commit-ci](https://github.com/pre-commit-ci))
+- chore: update pre-commit hooks [#1427](https://github.com/jupyter-server/jupyter_server/pull/1427) ([@pre-commit-ci](https://github.com/pre-commit-ci))
+
+### Documentation improvements
+
+- Update documentation for `cookie_secret` [#1433](https://github.com/jupyter-server/jupyter_server/pull/1433) ([@krassowski](https://github.com/krassowski))
+- Add Changelog for 2.14.1 [#1430](https://github.com/jupyter-server/jupyter_server/pull/1430) ([@blink1073](https://github.com/blink1073))
+- Update simple extension examples: \_jupyter_server_extension_points [#1426](https://github.com/jupyter-server/jupyter_server/pull/1426) ([@manics](https://github.com/manics))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter_server/graphs/contributors?from=2024-05-31&to=2024-07-12&type=c))
+
+[@blink1073](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Ablink1073+updated%3A2024-05-31..2024-07-12&type=Issues) | [@gogasca](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Agogasca+updated%3A2024-05-31..2024-07-12&type=Issues) | [@krassowski](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Akrassowski+updated%3A2024-05-31..2024-07-12&type=Issues) | [@manics](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Amanics+updated%3A2024-05-31..2024-07-12&type=Issues) | [@pre-commit-ci](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Apre-commit-ci+updated%3A2024-05-31..2024-07-12&type=Issues)
+
+<!-- <END NEW CHANGELOG ENTRY> -->
+
 ## 2.14.1
 
 ([Full Changelog](https://github.com/jupyter-server/jupyter_server/compare/v2.14.0...f1379164fa209bc4bfeadf43ab0e7f473b03a0ce))
@@ -27,8 +55,6 @@ All notable changes to this project will be documented in this file.
 
 [@blink1073](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Ablink1073+updated%3A2024-04-11..2024-05-31&type=Issues) | [@lresende](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Alresende+updated%3A2024-04-11..2024-05-31&type=Issues) | [@pre-commit-ci](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter_server+involves%3Apre-commit-ci+updated%3A2024-04-11..2024-05-31&type=Issues)
 
-<!-- <END NEW CHANGELOG ENTRY> -->
-
 ## 2.14.0
 
 ([Full Changelog](https://github.com/jupyter-server/jupyter_server/compare/v2.13.0...074628806d6b2ec3304d60ab5cfba1c326f67730))

jupyter_server/gateway/connections.py

@@ -47,6 +47,8 @@ async def connect(self):
             url_escape(self.kernel_id),
             "channels",
         )
+        if self.session_id:
+            ws_url += f"?session_id={url_escape(self.session_id)}"
         self.log.info(f"Connecting to {ws_url}")
         kwargs: dict[str, Any] = {}
         kwargs = GatewayClient.instance().load_connection_args(**kwargs)

jupyter_server/serverapp.py

@@ -1139,8 +1139,9 @@ def _default_cookie_secret_file(self) -> str:
         b"",
         config=True,
         help="""The random bytes used to secure cookies.
-        By default this is a new random number every time you start the server.
-        Set it to a value in a config file to enable logins to persist across server sessions.
+        By default this is generated on first start of the server and persisted across server
+        sessions by writing the cookie secret into the `cookie_secret_file` file.
+        When using an executable config file you can override this to be random at each server restart.
 
         Note: Cookie secrets should be kept private, do not share config files with
         cookie_secret stored in plaintext (you can read the value from a file).

jupyter_server/services/contents/handlers.py

@@ -140,7 +140,9 @@ async def get(self, path=""):
 
         hash_str = self.get_query_argument("hash", default="0")
         if hash_str not in {"0", "1"}:
-            raise web.HTTPError(400, f"Content {hash_str!r} is invalid")
+            raise web.HTTPError(
+                400, f"Hash argument {hash_str!r} is invalid. It must be '0' or '1'."
+            )
         require_hash = int(hash_str)
 
         if not cm.allow_hidden and await ensure_async(cm.is_hidden(path)):

jupyter_server/services/kernels/kernelmanager.py

@@ -232,18 +232,25 @@ async def _async_start_kernel(  # type:ignore[override]
                 kwargs["kernel_id"] = kernel_id
             kernel_id = await self.pinned_superclass._async_start_kernel(self, **kwargs)
             self._kernel_connections[kernel_id] = 0
-            task = asyncio.create_task(self._finish_kernel_start(kernel_id))
-            if not getattr(self, "use_pending_kernels", None):
-                await task
-            else:
-                self._pending_kernel_tasks[kernel_id] = task
+
             # add busy/activity markers:
             kernel = self.get_kernel(kernel_id)
             kernel.execution_state = "starting"  # type:ignore[attr-defined]
             kernel.reason = ""  # type:ignore[attr-defined]
             kernel.last_activity = utcnow()  # type:ignore[attr-defined]
             self.log.info("Kernel started: %s", kernel_id)
-            self.log.debug("Kernel args: %r", kwargs)
+            self.log.debug(
+                "Kernel args (excluding env): %r", {k: v for k, v in kwargs.items() if k != "env"}
+            )
+            env = kwargs.get("env", None)
+            if env and isinstance(env, dict):  # type:ignore[unreachable]
+                self.log.debug("Kernel argument 'env' passed with: %r", list(env.keys()))  # type:ignore[unreachable]
+
+            task = asyncio.create_task(self._finish_kernel_start(kernel_id))
+            if not getattr(self, "use_pending_kernels", None):
+                await task
+            else:
+                self._pending_kernel_tasks[kernel_id] = task
 
             # Increase the metric of number of kernels running
             # for the relevant kernel type by 1
@@ -532,6 +539,40 @@ def _check_kernel_id(self, kernel_id):
             raise web.HTTPError(404, "Kernel does not exist: %s" % kernel_id)
 
     # monitoring activity:
+    untracked_message_types = List(
+        trait=Unicode(),
+        config=True,
+        default_value=[
+            "comm_info_request",
+            "comm_info_reply",
+            "kernel_info_request",
+            "kernel_info_reply",
+            "shutdown_request",
+            "shutdown_reply",
+            "interrupt_request",
+            "interrupt_reply",
+            "debug_request",
+            "debug_reply",
+            "stream",
+            "display_data",
+            "update_display_data",
+            "execute_input",
+            "execute_result",
+            "error",
+            "status",
+            "clear_output",
+            "debug_event",
+            "input_request",
+            "input_reply",
+        ],
+        help="""List of kernel message types excluded from user activity tracking.
+
+        This should be a superset of the message types sent on any channel other
+        than the shell channel.""",
+    )
+
+    def track_message_type(self, message_type):
+        return message_type not in self.untracked_message_types
 
     def start_watching_activity(self, kernel_id):
         """Start watching IOPub messages on a kernel for activity.
@@ -552,15 +593,27 @@ def start_watching_activity(self, kernel_id):
 
         def record_activity(msg_list):
             """Record an IOPub message arriving from a kernel"""
-            self.last_kernel_activity = kernel.last_activity = utcnow()
-
             idents, fed_msg_list = session.feed_identities(msg_list)
             msg = session.deserialize(fed_msg_list, content=False)
 
             msg_type = msg["header"]["msg_type"]
+            parent_msg_type = msg.get("parent_header", {}).get("msg_type", None)
+            if (
+                self.track_message_type(msg_type)
+                or self.track_message_type(parent_msg_type)
+                or kernel.execution_state == "busy"
+            ):
+                self.last_kernel_activity = kernel.last_activity = utcnow()
             if msg_type == "status":
                 msg = session.deserialize(fed_msg_list)
-                kernel.execution_state = msg["content"]["execution_state"]
+                execution_state = msg["content"]["execution_state"]
+                if self.track_message_type(parent_msg_type):
+                    kernel.execution_state = execution_state
+                elif kernel.execution_state == "starting" and execution_state != "starting":
+                    # We always normalize post-starting execution state to "idle"
+                    # unless we know that the status is in response to one of our
+                    # tracked message types.
+                    kernel.execution_state = "idle"
                 self.log.debug(
                     "activity on %s: %s (%s)",
                     kernel_id,

tests/services/kernels/test_cull.py

@@ -1,7 +1,9 @@
 import asyncio
+import datetime
 import json
 import os
 import platform
+import uuid
 import warnings
 
 import jupyter_client
@@ -94,6 +96,83 @@ async def test_cull_idle(jp_fetch, jp_ws_fetch):
     assert culled
 
 
+@pytest.mark.parametrize(
+    "jp_server_config",
+    [
+        # Test the synchronous case
+        Config(
+            {
+                "ServerApp": {
+                    "kernel_manager_class": "jupyter_server.services.kernels.kernelmanager.MappingKernelManager",
+                    "MappingKernelManager": {
+                        "cull_idle_timeout": CULL_TIMEOUT,
+                        "cull_interval": CULL_INTERVAL,
+                        "cull_connected": True,
+                    },
+                }
+            }
+        ),
+        # Test the async case
+        Config(
+            {
+                "ServerApp": {
+                    "kernel_manager_class": "jupyter_server.services.kernels.kernelmanager.AsyncMappingKernelManager",
+                    "AsyncMappingKernelManager": {
+                        "cull_idle_timeout": CULL_TIMEOUT,
+                        "cull_interval": CULL_INTERVAL,
+                        "cull_connected": True,
+                    },
+                }
+            }
+        ),
+    ],
+)
+async def test_cull_connected(jp_fetch, jp_ws_fetch):
+    r = await jp_fetch("api", "kernels", method="POST", allow_nonstandard_methods=True)
+    kernel = json.loads(r.body.decode())
+    kid = kernel["id"]
+
+    # Open a websocket connection.
+    ws = await jp_ws_fetch("api", "kernels", kid, "channels")
+    session_id = uuid.uuid1().hex
+    message_id = uuid.uuid1().hex
+    await ws.write_message(
+        json.dumps(
+            {
+                "channel": "shell",
+                "header": {
+                    "date": datetime.datetime.now(tz=datetime.timezone.utc).isoformat(),
+                    "session": session_id,
+                    "msg_id": message_id,
+                    "msg_type": "execute_request",
+                    "username": "",
+                    "version": "5.2",
+                },
+                "parent_header": {},
+                "metadata": {},
+                "content": {
+                    "code": f"import time\ntime.sleep({CULL_TIMEOUT-1})",
+                    "silent": False,
+                    "allow_stdin": False,
+                    "stop_on_error": True,
+                },
+                "buffers": [],
+            }
+        )
+    )
+
+    r = await jp_fetch("api", "kernels", kid, method="GET")
+    model = json.loads(r.body.decode())
+    assert model["connections"] == 1
+    culled = await get_cull_status(
+        kid, jp_fetch
+    )  # connected, but code cell still running. Should not be culled
+    assert not culled
+    culled = await get_cull_status(kid, jp_fetch)  # still connected, but idle... should be culled
+    assert culled
+    ws.close()
+
+
 async def test_cull_idle_disable(jp_fetch, jp_ws_fetch, jp_kernelspec_with_metadata):
     r = await jp_fetch("api", "kernels", method="POST", allow_nonstandard_methods=True)
     kernel = json.loads(r.body.decode())