Merge branch 'main' into python3.13

Author: mathbunnyru

requirements-dev.txt

@@ -2,7 +2,6 @@ docker
 plumbum
 pre-commit
 pytest
-pytest-retry
 # `pytest-xdist` is a plugin that provides the `--numprocesses` flag,
 # allowing us to run `pytest` tests in parallel
 pytest-xdist

tests/by_image/all-spark-notebook/test_spark_notebooks.py

@@ -0,0 +1,41 @@
+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+from pathlib import Path
+
+import pytest  # type: ignore
+
+from tests.shared_checks.nbconvert_check import check_nbconvert
+from tests.utils.tracked_container import TrackedContainer
+
+LOGGER = logging.getLogger(__name__)
+THIS_DIR = Path(__file__).parent.resolve()
+
+
+@pytest.mark.parametrize(
+    "test_file,expected_warnings",
+    [
+        ("local_sparkR", ["WARNING: Using incubator modules: jdk.incubator.vector"]),
+        ("local_sparklyr", []),
+    ],
+)
+@pytest.mark.parametrize("output_format", ["pdf", "html", "markdown"])
+def test_spark_r_nbconvert(
+    container: TrackedContainer,
+    test_file: str,
+    output_format: str,
+    expected_warnings: list[str],
+) -> None:
+    host_data_file = THIS_DIR / "data" / f"{test_file}.ipynb"
+    logs = check_nbconvert(
+        container,
+        host_data_file,
+        output_format,
+        execute=True,
+        no_warnings=(not expected_warnings),
+    )
+
+    if expected_warnings:
+        warnings = TrackedContainer.get_warnings(logs)
+        assert len(warnings) == len(expected_warnings)
+        assert expected_warnings[0] == warnings[0]

tests/by_image/all-spark-notebook/test_spark_r_nbconvert.py

@@ -6,20 +6,20 @@
 import pytest  # type: ignore
 import requests
 
-from tests.utils.find_free_port import find_free_port
 from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
 
 
-def test_cli_args(container: TrackedContainer, http_client: requests.Session) -> None:
+def test_cli_args(
+    container: TrackedContainer, http_client: requests.Session, free_host_port: int
+) -> None:
     """Image should respect command line args (e.g., disabling token security)"""
-    host_port = find_free_port()
     container.run_detached(
         command=["start-notebook.py", "--IdentityProvider.token=''"],
-        ports={"8888/tcp": host_port},
+        ports={"8888/tcp": free_host_port},
     )
-    resp = http_client.get(f"http://localhost:{host_port}")
+    resp = http_client.get(f"http://localhost:{free_host_port}")
     resp.raise_for_status()
     logs = container.get_logs()
     LOGGER.debug(logs)
@@ -35,7 +35,7 @@ def test_nb_user_change(container: TrackedContainer) -> None:
     container.run_detached(
         user="root",
         environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"],
-        command=["bash", "-c", "sleep infinity"],
+        command=["sleep", "infinity"],
     )
 
     # Give the chown time to complete.
@@ -54,22 +54,21 @@ def test_nb_user_change(container: TrackedContainer) -> None:
 
 @pytest.mark.filterwarnings("ignore:Unverified HTTPS request")
 def test_unsigned_ssl(
-    container: TrackedContainer, http_client: requests.Session
+    container: TrackedContainer, http_client: requests.Session, free_host_port: int
 ) -> None:
     """Container should generate a self-signed SSL certificate
     and Jupyter Server should use it to enable HTTPS.
     """
-    host_port = find_free_port()
     container.run_detached(
         environment=["GEN_CERT=yes"],
-        ports={"8888/tcp": host_port},
+        ports={"8888/tcp": free_host_port},
     )
     # NOTE: The requests.Session backing the http_client fixture
     # does not retry properly while the server is booting up.
     # An SSL handshake error seems to abort the retry logic.
     # Forcing a long sleep for the moment until I have time to dig more.
     time.sleep(1)
-    resp = http_client.get(f"https://localhost:{host_port}", verify=False)
+    resp = http_client.get(f"https://localhost:{free_host_port}", verify=False)
     resp.raise_for_status()
     assert "login_submit" in resp.text
     logs = container.get_logs()
@@ -94,18 +93,18 @@ def test_unsigned_ssl(
 def test_custom_internal_port(
     container: TrackedContainer,
     http_client: requests.Session,
+    free_host_port: int,
     env: dict[str, str],
 ) -> None:
     """Container should be accessible from the host
     when using custom internal port"""
-    host_port = find_free_port()
     internal_port = env.get("JUPYTER_PORT", 8888)
     container.run_detached(
         command=["start-notebook.py", "--IdentityProvider.token=''"],
         environment=env,
-        ports={internal_port: host_port},
+        ports={internal_port: free_host_port},
     )
-    resp = http_client.get(f"http://localhost:{host_port}")
+    resp = http_client.get(f"http://localhost:{free_host_port}")
     resp.raise_for_status()
     logs = container.get_logs()
     LOGGER.debug(logs)

tests/by_image/base-notebook/test_container_options.py

@@ -35,11 +35,12 @@ def ipv6_network(docker_client: docker.DockerClient) -> Generator[str, None, Non
 
 def test_ipv46(container: TrackedContainer, ipv6_network: str) -> None:
     """Check server is listening on the expected IP families"""
-    host_data_dir = THIS_DIR / "data"
-    cont_data_dir = "/home/jovyan/data"
+    file_name = "check_listening.py"
+    host_file = THIS_DIR / "data" / file_name
+    cont_file = f"/home/jovyan/data/{file_name}"
     LOGGER.info("Testing that server is listening on IPv4 and IPv6 ...")
     container.run_detached(
         network=ipv6_network,
-        volumes={str(host_data_dir): {"bind": cont_data_dir, "mode": "ro,z"}},
+        volumes={host_file: {"bind": cont_file, "mode": "ro"}},
     )
-    container.exec_cmd(f"python {cont_data_dir}/check_listening.py")
+    container.exec_cmd(f"python {cont_file}")

tests/by_image/base-notebook/test_ips.py

@@ -2,16 +2,14 @@
 # Distributed under the terms of the Modified BSD License.
 import requests
 
-from tests.utils.find_free_port import find_free_port
 from tests.utils.tracked_container import TrackedContainer
 
 
 def test_secured_server(
-    container: TrackedContainer, http_client: requests.Session
+    container: TrackedContainer, http_client: requests.Session, free_host_port: int
 ) -> None:
     """Jupyter Server should eventually request user login."""
-    host_port = find_free_port()
-    container.run_detached(ports={"8888/tcp": host_port})
-    resp = http_client.get(f"http://localhost:{host_port}")
+    container.run_detached(ports={"8888/tcp": free_host_port})
+    resp = http_client.get(f"http://localhost:{free_host_port}")
     resp.raise_for_status()
     assert "login_submit" in resp.text, "User login not requested"

tests/by_image/base-notebook/test_notebook.py

@@ -6,7 +6,6 @@
 import pytest  # type: ignore
 import requests
 
-from tests.utils.find_free_port import find_free_port
 from tests.utils.tracked_container import TrackedContainer
 
 LOGGER = logging.getLogger(__name__)
@@ -32,6 +31,7 @@
 def test_start_notebook(
     container: TrackedContainer,
     http_client: requests.Session,
+    free_host_port: int,
     env: list[str] | None,
     expected_command: str,
     expected_start: bool,
@@ -41,8 +41,7 @@ def test_start_notebook(
     LOGGER.info(
         f"Test that the start-notebook.py launches the {expected_command} server from the env {env} ..."
     )
-    host_port = find_free_port()
-    container.run_detached(environment=env, ports={"8888/tcp": host_port})
+    container.run_detached(environment=env, ports={"8888/tcp": free_host_port})
     # sleeping some time to let the server start
     time.sleep(2)
     logs = container.get_logs()
@@ -59,7 +58,7 @@ def test_start_notebook(
     assert len(expected_warnings) == len(warnings)
     # checking if the server is listening
     if expected_start:
-        resp = http_client.get(f"http://localhost:{host_port}")
+        resp = http_client.get(f"http://localhost:{free_host_port}")
         assert resp.status_code == 200, "Server is not listening"
 
 

tests/by_image/base-notebook/test_start_container.py

@@ -1,8 +1,7 @@
 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-from tests.utils.run_command import run_command
 from tests.utils.tracked_container import TrackedContainer
 
 
 def test_julia(container: TrackedContainer) -> None:
-    run_command(container, "julia --version")
+    container.run_and_wait(timeout=5, command=["julia", "--version"])

tests/by_image/datascience-notebook/test_julia_datascience.py

@@ -7,7 +7,7 @@
 
 
 def test_pluto_proxy(
-    container: TrackedContainer, http_client: requests.Session
+    container: TrackedContainer, http_client: requests.Session, free_host_port: int
 ) -> None:
     """Pluto proxy starts Pluto correctly"""
-    check_pluto_proxy(container, http_client)
+    check_pluto_proxy(container, http_client, free_host_port)

tests/by_image/datascience-notebook/test_pluto_datascience.py

@@ -10,13 +10,14 @@
 LOGGER = logging.getLogger(__name__)
 
 
+@pytest.mark.parametrize("requested_only", [True, False])
 @pytest.mark.info
-def test_outdated_packages(
-    container: TrackedContainer, requested_only: bool = True
-) -> None:
+def test_outdated_packages(container: TrackedContainer, requested_only: bool) -> None:
     """Getting the list of updatable packages"""
     LOGGER.info(f"Checking outdated packages in {container.image_name} ...")
     pkg_helper = CondaPackageHelper(container)
-    pkg_helper.check_updatable_packages(requested_only)
-    LOGGER.info(pkg_helper.get_outdated_summary(requested_only))
-    LOGGER.info(f"\n{pkg_helper.get_outdated_table()}\n")
+    updatable = pkg_helper.find_updatable_packages(requested_only)
+    LOGGER.info(pkg_helper.get_outdated_summary(updatable, requested_only))
+    LOGGER.info(
+        f"Outdated packages table:\n{pkg_helper.get_outdated_table(updatable)}\n"
+    )