Upgrade to Python 3.12

.github/scripts/git_debug_image.sh

@@ -43,7 +43,7 @@ cd /home/user
 git clone -b "$BRANCH_NAME" https://github.com/RS-PYTHON/rs-server.git
 cd ./rs-server
 
-# The rspy modules used by the fastapi service are installed under /usr/local/lib/python3.11/site-packages
+# The rspy modules used by the fastapi service are installed under /usr/local/lib/python<version>/site-packages
 # We tell the service to use the git dirs instead. So we replace them by symlinks to git.
 py_site_packages=$(python -c "import site; print(site.getsitepackages()[0])")
 for git_pydir in $(pwd)/services/*/rs_server_*; do
@@ -53,7 +53,7 @@ for git_pydir in $(pwd)/services/*/rs_server_*; do
     fi
 done
 
-# Do the same for the files installed under /usr/local/lib/python3.11/site-packages/config and .../resources
+# Do the same for the files installed under /usr/local/lib/python<version>/site-packages/config and .../resources
 for dir in "config" "resources"; do
     if [ -d "$py_site_packages/$dir" ]; then
         for site_packages_file in "$py_site_packages/$dir"/*; do

.github/workflows/check-code-quality.yml

@@ -27,7 +27,7 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHON_VERSION: 3.11.7
+  PYTHON_VERSION: 3.12.9
   SONAR_PROJECT_KEY: RS-PYTHON_rs-server_AYw0m7ixvQv-JMsowILQ
 
 jobs:
@@ -199,7 +199,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/poetry-install
       - name: Run pytest
-        run: ./resources/all_pytests.sh
+        run: ./resources/all_pytests.sh --install
         shell: bash
 
       - name: Display code coverage summary in this console

.github/workflows/publish-binaries.yml

@@ -30,7 +30,7 @@ on:
         required: false
 
 env:
-  PYTHON_VERSION: 3.11.7
+  PYTHON_VERSION: 3.12.9
   DOCKER_REGISTRY: ghcr.io
 
 jobs:

pyproject.toml

@@ -20,7 +20,7 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3",
     "Topic :: Scientific/Engineering :: GIS",
     "Topic :: Scientific/Engineering :: Image Processing"
 ]
@@ -48,7 +48,7 @@ packages = [
 
 # All the dependencies for release are set in the sub-projects
 [tool.poetry.dependencies]
-python = "^3.11"
+python = ">=3.11,<4"
 rs-server-adgs = {path = "services/adgs", develop = true}
 rs-server-cadip = {path = "services/cadip", develop = true}
 rs-server-catalog = {path = "services/catalog", develop = true}

resources/all_pytests.sh

@@ -45,10 +45,12 @@ for toml in $(find "$ROOT_DIR" -name pyproject.toml | sort); do
     echo "Test '$tests_dir'"
 
     # Install dependencies
-    (set -x
-        cd "$proj_dir" && poetry -q install --with dev
-        poetry -q run opentelemetry-bootstrap -a install || true
-    )
+    if [[ " $@ " == *" --install "* ]]; then
+        (set -x
+            cd "$proj_dir" && poetry -q install --with dev
+            poetry -q run opentelemetry-bootstrap -a install || true
+        )
+    fi
 
     # Increment junit reports index
     junit=$((junit+1))

resources/build_base_images.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Copyright 2025 CS Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build the base Docker images that are used in the cluster and in the ci/cd.
+
+set -euo pipefail
+#set -x
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+BUILD_DIR="$(realpath $SCRIPT_DIR/build_base_images)"
+
+PYTHON_VERSION=3.13.2
+
+# For each dockerfile and associated docker image name, separated by a ;
+for params in \
+    "Dockerfile.python;python:${PYTHON_VERSION}-slim-bookworm" \
+    "Dockerfile.jupyter;quay.io/jupyter/base-notebook:hub-5.2.1"
+do
+    dockerfile=$(echo $params | cut -d ";" -f 1)
+    base=$(echo $params | cut -d ";" -f 2)
+
+    # Add our hosting github organization to the docker image
+    target="ghcr.io/rs-python/$base"
+
+    # Build the docker image
+    docker build \
+        --build-arg BASE=${base} \
+        --build-arg DASK_TAG=2025.3.0 \
+        --build-arg DASK_GATEWAY_TAG=2024.1.0 \
+        --build-arg PREFECT_TAG=3.2.13 \
+        --build-arg PREFECT_DASK_TAG=0.3.3 \
+        --progress plain \
+        -f "${BUILD_DIR}/${dockerfile}" \
+        -t "$target" \
+        "${BUILD_DIR}"
+
+    # Push the docker image to the registry, if the --push option is specified.
+    if [[ " $@ " == *" --push "* ]]; then
+        docker login https://ghcr.io/v2/rs-python
+        docker push "$target"
+    fi
+done

resources/build_base_images/Dockerfile.jupyter

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+# Copyright 2025 CS Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# quay.io/jupyter/base-notebook:<tag>
+ARG BASE=xxx
+FROM ${BASE}
+
+ENV PATH=/home/jovyan/.local/bin:${PATH}
+
+ARG DASK_TAG=xxx
+ARG DASK_GATEWAY_TAG=xxx
+ARG PREFECT_TAG=xxx
+ARG PREFECT_DASK_TAG=xxx
+
+COPY ./layer-cleanup.sh /usr/local/bin/
+
+USER root
+RUN chmod 755 /usr/local/bin/layer-cleanup.sh
+RUN apt update && apt upgrade -y && layer-cleanup.sh
+USER jovyan
+
+# Upgrade pip version
+RUN pip install -U pip && layer-cleanup.sh
+
+# Additional python kernel to install to run the notebooks
+ARG PYTHON_VERSION=3.11.7
+ARG KERNEL_NAME=py3.11.7
+
+# Install additional python kernel
+RUN conda create -y -n "$KERNEL_NAME" python="$PYTHON_VERSION" && \
+    source activate "$KERNEL_NAME" && \
+    python -V && \
+    pip install -U pip && \
+    pip install ipykernel && \
+    layer-cleanup.sh
+USER root
+RUN source activate "$KERNEL_NAME" && \
+    python -m ipykernel install --name "$KERNEL_NAME" --display-name "$KERNEL_NAME" && \
+    layer-cleanup.sh
+USER jovyan
+
+# Set default kernel... does not seem to work
+RUN jupyter notebook --generate-config && \
+    layer-cleanup.sh && \
+    echo "c.MappingKernelManager.default_kernel_name='$KERNEL_NAME'" >> /home/jovyan/.jupyter/jupyter_notebook_config.py
+
+# Note: put s3fs before boto3 to have a recent version
+RUN source activate "$KERNEL_NAME" && \
+    pip install \
+        dask[complete]=="${DASK_TAG}" \
+        distributed=="${DASK_TAG}" \
+        dask-gateway=="${DASK_GATEWAY_TAG}" \
+        prefect[aws]=="${PREFECT_TAG}" \
+        prefect-dask=="${PREFECT_DASK_TAG}" \
+        ipywidgets \
+        s3fs \
+        boto3 && \
+    layer-cleanup.sh
+
+# Install dot and clean conda
+USER root
+RUN apt update && apt install -y python3-pydot graphviz && layer-cleanup.sh
+USER jovyan
+
+# Use default python version from bash session
+RUN echo -e "\nsource activate $KERNEL_NAME" >> /home/jovyan/.bashrc
+
+USER root
+RUN chown -R jovyan /home/jovyan
+USER jovyan
+
+# Set labels based on the Open Containers Initiative (OCI):
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+#
+LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server"
+LABEL org.opencontainers.image.ref.name="ghcr.io/rs-python/quay.io/jupyter/base-notebook:hub-5.2.1"
+LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/resources/build_base_images/Dockerfile.jupyter.k8s"

resources/build_base_images/Dockerfile.python

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Copyright 2025 CS Group
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# python:<tag>
+ARG BASE=xxx
+FROM ${BASE}
+
+RUN apt update && apt upgrade -y
+
+# Upgrade pip version
+RUN pip install -U pip
+
+# Note: don't remove cache so the child images that use this one as a base will build faster
+# RUN rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
+
+# Set labels based on the Open Containers Initiative (OCI):
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+#
+LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server"
+LABEL org.opencontainers.image.ref.name="ghcr.io/rs-python/python:3.12.9-slim-bookworm"
+LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/resources/build_base_images/Dockerfile.python"