Merge branch 'main' into python3.13

mathbunnyru · web-flow · commit 22ab5926dbe6 · 2024-11-17T13:41:32.000Z
diff --git a/.gitignore b/.gitignore
@@ -106,8 +106,10 @@ ipython_config.py
 #pdm.lock
 #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
 #   in version control.
-#   https://pdm.fming.dev/#use-with-ide
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
 .pdm.toml
+.pdm-python
+.pdm-build/
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@
 repos:
   # Autoupdate: Python code
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
         args: [--py39-plus]
@@ -28,14 +28,14 @@ repos:
 
   # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: 24.8.0
+    rev: 24.10.0
     hooks:
       - id: black
         args: [--target-version=py39]
 
   # Check python code static typing
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.13.0
     hooks:
       - id: mypy
         args: [--config, ./mypy.ini]
@@ -69,7 +69,7 @@ repos:
 
   # `pre-commit sample-config` default hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
       - id: end-of-file-fixer
@@ -78,15 +78,15 @@ repos:
 
   # Lint: Dockerfile
   - repo: https://github.com/hadolint/hadolint
-    rev: v2.13.0-beta
+    rev: v2.13.1-beta
     hooks:
       - id: hadolint-docker
         entry: hadolint/hadolint:v2.12.1-beta hadolint
 
   # Lint: Dockerfile
   # We're linting .dockerfile files as well
   - repo: https://github.com/hadolint/hadolint
-    rev: v2.13.0-beta
+    rev: v2.13.1-beta
     hooks:
       - id: hadolint-docker
         name: Lint *.dockerfile Dockerfiles
@@ -124,14 +124,14 @@ repos:
 
   # Lint: Markdown
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.41.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint
         args: ["--fix"]
 
   # Strip output from Jupyter notebooks
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.7.1
+    rev: 0.8.0
     hooks:
       - id: nbstripout
 
@@ -149,7 +149,7 @@ repos:
 
   # Run black on python code blocks in documentation files.
   - repo: https://github.com/adamchainz/blacken-docs
-    rev: 1.18.0
+    rev: 1.19.1
     hooks:
       - id: blacken-docs
         # --skip-errors is added to allow us to have python syntax highlighting even if
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,17 +3,31 @@
 This changelog only contains breaking and/or significant changes manually introduced to this repository (using Pull Requests).
 All image manifests can be found in [the wiki](https://github.com/jupyter/docker-stacks/wiki).
 
+## 2024-11-08
+
+Affected: all images except `docker-stacks-foundation`.
+
+- **Breaking:** `base-notebook`: stop installing `nodejs` from `conda-forge` ([#2172](https://github.com/jupyter/docker-stacks/pull/2172)).
+
+  Reason: It isn't a direct dependency on anything in the images any more, and increased the image size with ~150MB.
+
+## 2024-11-06
+
+Affected: all images except `docker-stacks-foundation`.
+
+- **Non-breaking:** `base-notebook`: install `jupyterhub-base` and `nodejs` packages instead of `jupyterhub` package ([#2171](https://github.com/jupyter/docker-stacks/pull/2171)).
+
 ## 2024-10-23
 
 Affected: all images.
 
-- **Breaking:** Switch to Python 3.12 ([#2072](https://github.com/jupyter/docker-stacks/pull/2072)).
+- **Breaking:** `docker-stacks-foundation`: switch to Python 3.12 ([#2072](https://github.com/jupyter/docker-stacks/pull/2072)).
 
 ## 2024-10-22
 
 Affected: `pyspark-notebook` and `all-spark-notebook` images.
 
-- **Breaking:** Start using Spark 4.0.0 preview versions ([#2159](https://github.com/jupyter/docker-stacks/pull/2159)).
+- **Breaking:** `pyspark-notebook`: start using Spark 4.0.0 preview versions ([#2159](https://github.com/jupyter/docker-stacks/pull/2159)).
   `sparklyr` doesn't seem to support Spark v4 yet when using Spark locally.
 
   Reason: Spark v3 is not compatible with Python 3.12, and [the voting group has decided](https://github.com/jupyter/docker-stacks/pull/2072#issuecomment-2414123851) to switch to Spark v4 preview version.
@@ -22,4 +36,4 @@ Affected: `pyspark-notebook` and `all-spark-notebook` images.
 
 Affected: users building a custom set of images.
 
-- **Breaking:** Rename: `ROOT_CONTAINER`->`ROOT_IMAGE`, `BASE_CONTAINER`->`BASE_IMAGE` ([#2154](https://github.com/jupyter/docker-stacks/issues/2154), [#2155](https://github.com/jupyter/docker-stacks/pull/2155)).
+- **Breaking:** rename: `ROOT_CONTAINER`->`ROOT_IMAGE`, `BASE_CONTAINER`->`BASE_IMAGE` ([#2154](https://github.com/jupyter/docker-stacks/issues/2154), [#2155](https://github.com/jupyter/docker-stacks/pull/2155)).
diff --git a/README.md b/README.md
@@ -1,7 +1,6 @@
 # Jupyter Docker Stacks
 
-[![GitHub actions badge](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml/badge.svg)
-](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml?query=branch%3Amain "Docker images build status")
+[![GitHub actions badge](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml/badge.svg)](https://github.com/jupyter/docker-stacks/actions/workflows/docker.yml?query=branch%3Amain "Docker images build status")
 [![Read the Docs badge](https://img.shields.io/readthedocs/jupyter-docker-stacks.svg)](https://jupyter-docker-stacks.readthedocs.io/en/latest/ "Documentation build status")
 [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/jupyter/docker-stacks/main.svg)](https://results.pre-commit.ci/latest/github/jupyter/docker-stacks/main "pre-commit.ci build status")
 [![Discourse badge](https://img.shields.io/discourse/users.svg?color=%23f37626&server=https%3A%2F%2Fdiscourse.jupyter.org)](https://discourse.jupyter.org/ "Jupyter Discourse Forum")
diff --git a/docs/conf.py b/docs/conf.py
@@ -67,6 +67,7 @@
     r"http://127\.0\.0\.1:.*",  # various examples
     r"https://mybinder\.org/v2/gh/.*",  # lots of 500 errors
     r"https://packages\.ubuntu\.com/search\?keywords=openjdk",  # frequent read timeouts
+    r"https://anaconda\.org\/conda-forge",  # frequent read timeouts
 ]
 
 linkcheck_allowed_redirects = {
diff --git a/docs/maintaining/aarch64-runner.md b/docs/maintaining/aarch64-runner.md
@@ -7,7 +7,7 @@ Each runner is recommended to have at least _2 cores_ and _30 GB_ of disk space.
 Add a new runner:
 
 - To use [Oracle OCI](https://www.oracle.com/cloud-0/), create a compute instance `VM.Standard.A1.Flex`.
-- To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-arm-vm-instance#armpublicimage).
+- To use [Google Cloud](https://cloud.google.com), use [this instruction](https://cloud.google.com/compute/docs/instances/create-start-instance).
 
 Configure your runner:
 
diff --git a/docs/using/recipe_code/jupyterhub_version.dockerfile b/docs/using/recipe_code/jupyterhub_version.dockerfile
@@ -1,6 +1,6 @@
 FROM quay.io/jupyter/base-notebook
 
-RUN mamba install --yes 'jupyterhub==4.0.1' && \
+RUN mamba install --yes 'jupyterhub-singleuser==4.0.1' && \
     mamba clean --all -f -y && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
diff --git a/docs/using/selecting.md b/docs/using/selecting.md
@@ -54,7 +54,7 @@ It contains:
 
 - Everything in `jupyter/docker-stacks-foundation`
 - Minimally functional Server (e.g., no LaTeX support for saving notebooks as PDFs)
-- `notebook`, `jupyterhub`, and `jupyterlab` packages
+- `notebook`, `jupyterhub-singleuser`, and `jupyterlab` packages
 - A `start-notebook.py` script as the default command
 - A `start-singleuser.py` script useful for launching containers in JupyterHub
 - Options for a self-signed HTTPS certificate
diff --git a/docs/using/specifics.md b/docs/using/specifics.md
@@ -49,7 +49,8 @@ You can build a `pyspark-notebook` image with a different `Spark` version by ove
     - This version needs to match the version supported by the Spark distribution used above.
     - See [Spark Overview](https://spark.apache.org/docs/latest/#downloading) and [Ubuntu packages](https://packages.ubuntu.com/search?keywords=openjdk).
   - `spark_version` (optional): The Spark version to install, for example `3.5.0`.
-    If not specified (this is the default), latest stable Spark will be installed.
+    If not specified (this is the default), latest Spark will be installed.
+    Note: to support Python 3.12, we currently install Spark v4 preview versions: <https://github.com/jupyter/docker-stacks/pull/2072#issuecomment-2414123851>.
   - `hadoop_version`: The Hadoop version (`3` by default).
     Note, that _Spark < 3.3_ require to specify `major.minor` Hadoop version (i.e. `3.2`).
   - `scala_version` (optional): The Scala version, for example `2.13` (not specified by default).
diff --git a/images/base-notebook/Dockerfile b/images/base-notebook/Dockerfile
@@ -39,13 +39,16 @@ USER ${NB_UID}
 # files across image layers when the permissions change
 WORKDIR /tmp
 RUN mamba install --yes \
-    'jupyterhub' \
+    'jupyterhub-singleuser' \
     'jupyterlab' \
     'nbclassic' \
-    'notebook' && \
+    # Sometimes, when the new version of `jupyterlab` is released, latest `notebook` might not support it for some time
+    # Old versions of `notebook` (<v7) didn't have a restriction on the `jupyterlab` version, and old `notebook` is getting installed
+    # That's why we have to pin the minimum notebook version
+    # More info: https://github.com/jupyter/docker-stacks/pull/2167
+    'notebook>=7.2.2' && \
     jupyter server --generate-config && \
     mamba clean --all -f -y && \
-    npm cache clean --force && \
     jupyter lab clean && \
     rm -rf "/home/${NB_USER}/.cache/yarn" && \
     fix-permissions "${CONDA_DIR}" && \
diff --git a/images/pyspark-notebook/Dockerfile b/images/pyspark-notebook/Dockerfile
@@ -24,7 +24,7 @@ RUN apt-get update --yes && \
     ca-certificates-java && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
-# If spark_version is not set, latest stable Spark will be installed
+# If spark_version is not set, latest Spark will be installed
 ARG spark_version
 ARG hadoop_version="3"
 # If scala_version is not set, Spark without Scala will be installed
diff --git a/images/pyspark-notebook/setup_spark.py b/images/pyspark-notebook/setup_spark.py
@@ -29,11 +29,11 @@ def get_all_refs(url: str) -> list[str]:
 
 def get_latest_spark_version() -> str:
     """
-    Returns the last stable version of Spark using spark archive
+    Returns the last version of Spark using spark archive
     """
     LOGGER.info("Downloading Spark versions information")
     all_refs = get_all_refs("https://archive.apache.org/dist/spark/")
-    stable_versions = [
+    versions = [
         ref.removeprefix("spark-").removesuffix("/")
         for ref in all_refs
         if ref.startswith("spark-") and "incubating" not in ref
@@ -49,7 +49,7 @@ def version_array(ver: str) -> tuple[int, int, int, str]:
         patch, _, preview = arr[2].partition("-")
         return (major, minor, int(patch), preview)
 
-    latest_version = max(stable_versions, key=lambda ver: version_array(ver))
+    latest_version = max(versions, key=lambda ver: version_array(ver))
     LOGGER.info(f"Latest version: {latest_version}")
     return latest_version
 
diff --git a/images/pytorch-notebook/cuda12/Dockerfile b/images/pytorch-notebook/cuda12/Dockerfile
@@ -13,7 +13,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Install PyTorch with pip (https://pytorch.org/get-started/locally/)
 # hadolint ignore=DL3013
-RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu121' \
+RUN pip install --no-cache-dir --extra-index-url=https://pypi.nvidia.com --index-url 'https://download.pytorch.org/whl/cu124' \
     'torch' \
     'torchaudio' \
     'torchvision' && \
diff --git a/images/tensorflow-notebook/cuda/Dockerfile b/images/tensorflow-notebook/cuda/Dockerfile
@@ -14,7 +14,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 # Install TensorFlow, CUDA and cuDNN with pip
 RUN pip install --no-cache-dir \
     "jupyter-server-proxy" \
-    "tensorflow[and-cuda]" && \
+    "tensorflow[and-cuda]<=2.17.1" && \
     fix-permissions "${CONDA_DIR}" && \
     fix-permissions "/home/${NB_USER}"
 
diff --git a/tests/base-notebook/test_npm_package_manager.py b/tests/base-notebook/test_npm_package_manager.py
diff --git a/tests/docker-stacks-foundation/test_packages.py b/tests/docker-stacks-foundation/test_packages.py
@@ -74,8 +74,10 @@
     "grpcio-status",
     "grpcio",
     "hdf5",
+    "jupyterhub-singleuser",
     "jupyterlab-git",
     "mamba[version='<2.0.0']",
+    "notebook[version='>",
     "openssl",
     "pandas[version='>",
     "protobuf",

Original file line number	Diff line number	Diff line change
`@@ -67,6 +67,7 @@`
`67`	`67`	`r"http://127\.0\.0\.1:.*", # various examples`
`68`	`68`	`r"https://mybinder\.org/v2/gh/.*", # lots of 500 errors`
`69`	`69`	`r"https://packages\.ubuntu\.com/search\?keywords=openjdk", # frequent read timeouts`
	`70`	`+ r"https://anaconda\.org\/conda-forge", # frequent read timeouts`
`70`	`71`	`]`
`71`	`72`
`72`	`73`	`linkcheck_allowed_redirects = {`