move more shared code into scripts

.github/workflows/release-to-nvstaging.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2025, NVIDIA CORPORATION.
+# Copyright (c) 2024-2026, NVIDIA CORPORATION.
 
 name: Publish release images to NGC
 

.github/workflows/test-notebooks.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2023-2025, NVIDIA CORPORATION.
+# Copyright (c) 2023-2026, NVIDIA CORPORATION.
 name: Test notebooks
 
 on:
@@ -45,9 +45,6 @@ permissions:
   security-events: none
   statuses: none
 
-env:
-  GHA_TOOLS_DIR: /home/rapids/.local/bin # Workaround for https://github.com/actions/runner/issues/2411
-
 jobs:
   test:
     strategy:
@@ -77,20 +74,18 @@ jobs:
           mamba install -n base --freeze-installed \
             curl \
             git
-      - name: Install gha-tools
-        run: |
-          mkdir -p "$GHA_TOOLS_DIR"
-          curl -s -L 'https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz' | tar -xz -C "$GHA_TOOLS_DIR"
       - name: Checkout code
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
+      - name: Install gha-tools
+        run: |
+          context/scripts/install-gha-tools
       - name: Get RAPIDS GitHub Info
         id: get-rapids-github-info
         uses: rapidsai/shared-actions/rapids-github-info@main
       - name: Print environment
         run: |
-          PATH="$PATH:$GHA_TOOLS_DIR"
           rapids-print-env
           rapids-logger "nvidia-smi"
           nvidia-smi
@@ -99,7 +94,6 @@ jobs:
       - name: Install awscli
         if: '!cancelled()'
         run: |
-          PATH="$PATH:$GHA_TOOLS_DIR"
           rapids-mamba-retry install -n base awscli
       - uses: aws-actions/configure-aws-credentials@v5
         if: '!cancelled()'
@@ -110,5 +104,4 @@ jobs:
       - name: Upload notebook test outputs
         if: '!cancelled()'
         run: |
-          PATH="$PATH:$GHA_TOOLS_DIR"
           rapids-upload-to-s3 test_notebooks_output_${{ inputs.ARCH }}_cuda${{ inputs.CUDA_VER }}_py${{ inputs.PYTHON_VER }}_${{ inputs.GPU }}-${{ inputs.DRIVER }}.tar.gz /home/rapids/notebooks_output

Dockerfile

@@ -65,22 +65,11 @@ ARG MINIFORGE_VER=notset
 
 FROM condaforge/miniforge3:${MINIFORGE_VER} AS miniforge-upstream
 
-SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
-
-RUN <<EOF
-# Ensure new files/dirs have group write permissions
-umask 002
-
-# install gha-tools for rapids-mamba-retry
-wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin
-
-# Example of pinned package in case you require an override
-# echo '<PACKAGE_NAME>==<VERSION>' >> /opt/conda/conda-meta/pinned
-
-# update everything before other environment changes, to ensure mixing
-# an older conda with newer packages still works well
-PATH="/opt/conda/bin:$PATH" \
-  rapids-mamba-retry update --all -y -n base
+RUN \
+  --mount=type=bind,source=scripts,target=/tmp/build-scripts \
+<<EOF
+  # update everything in 'base' before we copy files into later targets
+  /tmp/build-scripts/update-base-conda-environment
 EOF
 
 ################################ build miniforge-cuda using updated miniforge-upstream from above ###############################
@@ -96,24 +85,6 @@ ENV PYTHON_VERSION=${PYTHON_VER}
 
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
-# Set apt policy configurations
-# We bump up the number of retries and the timeouts for `apt`
-RUN <<EOF
-echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors
-echo 'APT::Acquire::Retries "10";' > /etc/apt/apt.conf.d/retries
-echo 'APT::Acquire::https::Timeout "240";' > /etc/apt/apt.conf.d/https-timeout
-echo 'APT::Acquire::http::Timeout "240";' > /etc/apt/apt.conf.d/http-timeout
-EOF
-
-# Install gha-tools
-RUN <<EOF
-  i=0; until apt-get update -y; do ((++i >= 5)) && break; sleep 10; done
-  apt-get install -y --no-install-recommends wget
-  wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin
-  apt-get purge -y wget && apt-get autoremove -y
-  rm -rf /var/lib/apt/lists/*
-EOF
-
 # Create a conda group and assign it as root's primary group
 RUN <<EOF
 groupadd conda
@@ -123,69 +94,20 @@ EOF
 # Ownership & permissions based on https://docs.anaconda.com/anaconda/install/multi-user/#multi-user-anaconda-installation-on-linux
 COPY --from=miniforge-upstream --chown=root:conda --chmod=770 /opt/conda /opt/conda
 
-RUN <<EOF
-# Ensure new files are created with group write access & setgid. See https://unix.stackexchange.com/a/12845
-chmod g+ws /opt/conda
-
-# Ensure new files/dirs have group write permissions
-umask 002
-
-# install expected Python version
-PYTHON_MAJOR_VERSION=${PYTHON_VERSION%%.*}
-PYTHON_MINOR_VERSION=${PYTHON_VERSION#*.}
-PYTHON_UPPER_BOUND="${PYTHON_MAJOR_VERSION}.$((PYTHON_MINOR_VERSION+1)).0a0"
-PYTHON_MINOR_PADDED=$(printf "%02d" "$PYTHON_MINOR_VERSION")
-PYTHON_VERSION_PADDED="${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_PADDED}"
-# 'shellcheck' is unhappy with the use of '>' to compare decimals here, but it works as expected for the 'bash' version in these
-# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose.
-#
-# shellcheck disable=SC2072
-if [[ "$PYTHON_VERSION_PADDED" > "3.12" ]]; then
-    PYTHON_ABI_TAG="cp${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION}"
-else
-    PYTHON_ABI_TAG="cpython"
-fi
-rapids-mamba-retry install -y -n base "python>=${PYTHON_VERSION},<${PYTHON_UPPER_BOUND}=*_${PYTHON_ABI_TAG}"
-rapids-mamba-retry update --all -y -n base
-find /opt/conda -follow -type f -name '*.a' -delete
-find /opt/conda -follow -type f -name '*.pyc' -delete
-# recreate missing libstdc++ symlinks
-conda clean -aiptfy
-EOF
-
-# Reassign root's primary group to root
-RUN usermod -g root root
-
-RUN <<EOF
-# ensure conda environment is always activated
-ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh
-echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/skel/.bashrc
-echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> ~/.bashrc
-EOF
+RUN \
+  --mount=type=bind,source=scripts,target=/tmp/build-scripts \
+<<EOF
+# configure apt (do this first because it affects installs in later scripts)
+/tmp/build-scripts/configure-apt
 
-# tzdata is needed by the ORC library used by pyarrow, because it provides /etc/localtime
-# On Ubuntu 24.04 and newer, we also need tzdata-legacy
-RUN <<EOF
+# install gha-tools
+/tmp/build-scripts/install-gha-tools
 
-PACKAGES_TO_INSTALL=(
-  tzdata
-)
+# set up conda
+/tmp/build-scripts/configure-conda-base-environment
 
-os_version=$(grep 'VERSION_ID' /etc/os-release | cut -d '"' -f 2)
-# 'shellcheck' is unhappy with the use of '>' to compare decimals here, but it works as expected for the 'bash' version in these
-# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose.
-#
-# shellcheck disable=SC2072
-if [[ "${os_version}" > "24.04" ]] || [[ "${os_version}" == "24.04" ]]; then
-    PACKAGES_TO_INSTALL+=(tzdata-legacy)
-fi
-
-rapids-retry apt-get update -y
-apt-get upgrade -y
-apt-get install -y --no-install-recommends \
-  "${PACKAGES_TO_INSTALL[@]}"
-
-rm -rf "/var/lib/apt/lists/*"
+# install tzdata system packages
+/tmp/build-scripts/install-tzdata-packages
 EOF
 
 # --- end 'rapidsai/miniforge-cuda' --- #
@@ -198,7 +120,10 @@ ARG RAPIDS_VER=26.04
 
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
-RUN <<EOF
+RUN \
+  --mount=type=bind,source=scripts,target=/tmp/build-scripts \
+<<EOF
+# install a few other system packages needed by 'rapidsai/base' users
 apt-get update
 PACKAGES_TO_INSTALL=(
   curl
@@ -207,7 +132,10 @@ PACKAGES_TO_INSTALL=(
 )
 apt-get install -y --no-install-recommends \
   "${PACKAGES_TO_INSTALL[@]}"
-curl --silent --show-error -L https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz | tar -xz -C /usr/local/bin
+
+# install gha-tools
+/tmp/build-scripts/install-gha-tools
+
 rm -rf /var/lib/apt/lists/*
 EOF
 

context/.dockerignore

@@ -4,4 +4,9 @@
 !cuvs-bench/
 !entrypoint.sh
 !notebooks.sh
+!scripts/configure-apt
+!scripts/configure-conda-base-environment
+!scripts/install-gha-tools
+!scripts/install-tzdata-packages
+!scripts/update-base-conda-environment
 !test_notebooks.py

context/scripts/README.md

@@ -0,0 +1,7 @@
+# context/scripts
+
+Some images in this repo have similar needs and their Dockerfiles require similar steps.
+
+This directory holds scripts intended to be shared among them, to reduce code duplication.
+
+For background, see https://github.com/rapidsai/docker/pull/836#discussion_r2674112913

context/scripts/configure-apt

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# [description]
+#
+#   Updates apt's settings.
+#   Initially introduced to add more and longer retries, to try to get past network issues.
+#
+
+set -e -u -o pipefail
+
+# Set apt policy configurations
+# We bump up the number of retries and the timeouts for `apt`
+echo 'APT::Update::Error-Mode "any";' > /etc/apt/apt.conf.d/warnings-as-errors
+echo 'APT::Acquire::Retries "10";' > /etc/apt/apt.conf.d/retries
+echo 'APT::Acquire::https::Timeout "240";' > /etc/apt/apt.conf.d/https-timeout
+echo 'APT::Acquire::http::Timeout "240";' > /etc/apt/apt.conf.d/http-timeout

context/scripts/configure-conda-base-environment

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# [description]
+#
+#   Configures the image's environment so users can be productive in the 'base' conda env.
+#   For example:
+#
+#     * installs the expected Python version
+#     * ensures the 'base' environment is always activated by default
+#
+
+set -e -u -o pipefail
+
+# Ensure new files are created with group write access & setgid. See https://unix.stackexchange.com/a/12845
+chmod g+ws /opt/conda
+
+# Ensure new files/dirs have group write permissions
+umask 002
+
+# install expected Python version
+PYTHON_MAJOR_VERSION=${PYTHON_VERSION%%.*}
+PYTHON_MINOR_VERSION=${PYTHON_VERSION#*.}
+PYTHON_UPPER_BOUND="${PYTHON_MAJOR_VERSION}.$((PYTHON_MINOR_VERSION+1)).0a0"
+PYTHON_MINOR_PADDED=$(printf "%02d" "$PYTHON_MINOR_VERSION")
+PYTHON_VERSION_PADDED="${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_PADDED}"
+
+# 'shellcheck' is unhappy with the use of '>' to compare decimals here, but it works as expected for the 'bash' version in these
+# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose.
+#
+# shellcheck disable=SC2072
+if [[ "$PYTHON_VERSION_PADDED" > "3.12" ]]; then
+    PYTHON_ABI_TAG="cp${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION}"
+else
+    PYTHON_ABI_TAG="cpython"
+fi
+rapids-mamba-retry install -y -n base "python>=${PYTHON_VERSION},<${PYTHON_UPPER_BOUND}=*_${PYTHON_ABI_TAG}"
+rapids-mamba-retry update --all -y -n base
+find /opt/conda -follow -type f -name '*.a' -delete
+find /opt/conda -follow -type f -name '*.pyc' -delete
+
+# recreate missing libstdc++ symlinks
+conda clean -aiptfy
+
+# Reassign root's primary group to root
+usermod -g root root
+
+# ensure conda environment is always activated
+ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh
+echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/skel/.bashrc
+echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> ~/.bashrc

context/scripts/install-gha-tools

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# [description]
+#
+#   Installs 'gha-tools' and puts it at /usr/local/bin so no PATH modification is required.
+#
+
+set -e -u -o pipefail
+
+GHA_TOOLS_INSTALL_DIR=/usr/local/bin
+
+# install wget
+wget_preinstalled=true
+
+if ! type -f wget >/dev/null; then
+  echo "could not find 'wget', installing it"
+  wget_preinstalled=false
+  i=0
+  until apt-get update -y; do
+    ((++i >= 5)) && break;
+    sleep 10;
+  done
+  apt-get install -y --no-install-recommends \
+    wget
+fi
+
+# install gha-tools
+wget -q https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C "${GHA_TOOLS_INSTALL_DIR}"
+
+# uninstall wget
+if [[ "${wget_preinstalled}" != "true" ]]; then
+  apt-get purge -y wget
+  apt-get autoremove -y
+  rm -rf /var/lib/apt/lists/*
+fi

context/scripts/install-tzdata-packages

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# [description]
+#
+#   tzdata is needed by the ORC library used by pyarrow, because it provides /etc/localtime
+#   On Ubuntu 24.04 and newer, we also need tzdata-legacy
+#
+
+set -e -u -o pipefail
+
+PACKAGES_TO_INSTALL=(
+  tzdata
+)
+
+os_version=$(grep 'VERSION_ID' /etc/os-release | cut -d '"' -f 2)
+
+# 'shellcheck' is unhappy with the use of '>' to compare decimals here, but it works as expected for the 'bash' version in these
+# images, and installing 'bc' or using a Python interpreter seem heavy for this purpose.
+#
+# shellcheck disable=SC2072
+if [[ "${os_version}" > "24.04" ]] || [[ "${os_version}" == "24.04" ]]; then
+    PACKAGES_TO_INSTALL+=(tzdata-legacy)
+fi
+
+rapids-retry apt-get update -y
+apt-get upgrade -y
+apt-get install -y --no-install-recommends \
+  "${PACKAGES_TO_INSTALL[@]}"
+
+rm -rf "/var/lib/apt/lists/*"