Merge pull request #857 from RS-PYTHON/feat/rspy570-empty-dpr-processor

feat: build dask-gateway + rs-server-staging image

Author: jgaucher-cs

.github/actions/publish-docker/action.yml

@@ -22,6 +22,9 @@ inputs:
   build_context_path:
     description: "'docker build' context path"
     required: true
+  image_basename:
+    description: Docker image base name
+    default: ${{ github.repository }} # = owner/repository-name
   image_suffix:
     description: Docker image name suffix e.g. _services-cadip
     required: true
@@ -46,6 +49,9 @@ inputs:
   debug_mode:
     description: Add debug tools to the docker image
     required: false
+  build_args:
+    description: List of build-time variables
+    required: false
 
 outputs:
   docker_image:
@@ -78,7 +84,7 @@ runs:
     # Full Docker image name:tag as ghcr.io/RS-PYTHON/rs-server<suffix>:<version> in lowercase
     - id: docker_image
       run: |
-        docker_image=${{ env.DOCKER_REGISTRY }}/${{ github.repository }}${{ inputs.image_suffix }}:${{ env.docker_version_name }}
+        docker_image=${{ env.DOCKER_REGISTRY }}/${{ inputs.image_basename }}${{ inputs.image_suffix }}:${{ env.docker_version_name }}
         docker_image=${docker_image,,} # lowercase
         echo docker_image=${docker_image} >> $GITHUB_ENV
         echo docker_image=${docker_image} >> $GITHUB_OUTPUT
@@ -130,6 +136,7 @@ runs:
         tags: ${{ env.docker_image }}
         labels: ${{ steps.meta.outputs.labels }}
         push: false # push after the security scans below
+        build-args: ${{ inputs.build_args }}
 
     - name: Run Trivy vulnerability scanner
       uses: aquasecurity/trivy-action@master

.github/workflows/publish-binaries.yml

@@ -261,15 +261,6 @@ jobs:
           fetch-depth: 0 # so that Dunamai produce the correct version
       - uses: ./.github/actions/install-python
 
-      # To pull ghcr.io/rs-python/apikey-manager
-      # TODO: to be changed when the apikey manager will have its own container registry.
-      - name: Log into Docker registry
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.DOCKER_REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Generate the aggregated openapi.json/swagger
         run: |
           set -x
@@ -524,6 +515,50 @@ jobs:
           branch_name: ${{ needs.set-env.outputs.branch_name }}
           debug_mode: ${{ needs.set-env.outputs.debug_mode }}
 
+  dask-staging-local-img:
+    if: github.actor != 'dependabot[bot]'
+    runs-on: ubuntu-latest
+    name: "'dask-gateway-server/staging/local' Docker image"
+    # TODO: this has to be activated when the pydantic conflict is solved
+    #needs: [set-env, services-common-whl, services-staging-whl]
+    # TODO: this has to be deleted when the pydantic conflict is solved
+    needs: [set-env, services-staging-whl]
+    permissions: write-all
+    outputs:
+      docker_image: ${{ steps.publish-docker.outputs.docker_image}}
+    steps:
+      - uses: actions/checkout@v4
+
+      # Download .whl files into a local dir
+      # TODO: the next 5 commented lines must be reactivated when the pydantic conflict is solved
+      #- name: Download .whl dependencies
+      #  uses: actions/download-artifact@v4
+      #  with:
+      #    name: ${{ needs.services-common-whl.outputs.package_name }}
+      #    path: ./whl
+      - name: Download .whl dependencies
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.services-staging-whl.outputs.package_name }}
+          path: ./build_context_path
+
+      - id: publish-docker
+        uses: ./.github/actions/publish-docker
+        with:
+          dockerfile: ./services/staging/.github/Dockerfile.dask-staging-local
+          build_context_path: ./build_context_path
+          image_basename: "${{ github.repository_owner }}/dask-gateway-server"
+          image_suffix: /staging/local
+          version_name: ${{ needs.services-staging-whl.outputs.version_name }}
+          dockerhub_username: ${{ vars.DOCKERHUB_USERNAME }}
+          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          docker_tag: ${{ needs.set-env.outputs.docker_tag }}
+          branch_name: ${{ needs.set-env.outputs.branch_name }}
+          debug_mode: false
+          build_args: |
+            DASK_GATEWAY_TAG=2024.1.0
+
   ###########
   # TESTING #
   ###########

poetry.lock

@@ -23,6 +23,11 @@ images=\
 "jupyter/minimal-notebook:latest "\
 "quay.io/jupyter/base-notebook:hub-4.1.5"\
 
+# Target images will be:
+# ghcr.io/rs-python/python:3.11.7-slim-bookworm
+# ghcr.io/rs-python/jupyter/minimal-notebook:latest
+# ghcr.io/rs-python/quay.io/jupyter/base-notebook:hub-4.1.5
+
 dockerdir="/tmp/dockerfile"
 dockerfile="$dockerdir/Dockerfile"
 mkdir -p "$dockerdir"
@@ -34,23 +39,64 @@ for image in $images; do
     # Save the default user in the image
     user=$(docker run --rm --entrypoint whoami "$image")
 
+    # Add our hosting github organization to the docker image
+    target="ghcr.io/rs-python/$image"
+
     # Create a tmp Dockerfile that pulls and update the image.
     cat << EOF > "$dockerfile"
 FROM $image
 USER root
 RUN apt update && apt upgrade -y
 USER $user
 
+# Upgrade pip version
+RUN pip install -U pip
+
+# Set labels based on the Open Containers Initiative (OCI):
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+#
+LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server"
+LABEL org.opencontainers.image.ref.name="$target"
+LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/resources/apt_upgrade_images.sh"
+
 # Note: don't remove cache so the child images that use this one as a base will build faster
-# RUN rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
+# RUN rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 EOF
 
-    cat "$dockerfile"
+    # For the jupyter images
+    if [[ $image == *"jupyter"* ]]; then
 
-    # Add our hosting github organization to the docker image
-    target="ghcr.io/rs-python/$image"
+        DASK_TAG=2024.5.2
+        DASK_GATEWAY_TAG=2024.1.0
+        PREFECT_TAG=3.1.4
+
+        cat << EOF >> "$dockerfile"
+
+# Install python 3.11.7 using conda then prefect and dask and other packages.
+# The versions must be the same than the cluster images.
+RUN conda install --yes conda-forge::python="3.11.7"
+
+# Note: put s3fs before boto3 to have a recent version
+RUN pip install \
+        dask[complete]=="${DASK_TAG}" \
+        distributed=="${DASK_TAG}" \
+        dask-gateway=="${DASK_GATEWAY_TAG}" \
+        prefect[dask,aws]=="${PREFECT_TAG}" \
+        ipywidgets \
+        s3fs \
+        boto3
+
+# Install dot and clean conda
+USER root
+RUN apt install -y python3-pydot graphviz
+RUN conda clean --all --yes
+USER jovyan
+EOF
+    fi
+
+    cat "$dockerfile"
 
     # Build and publish the image
-    docker build -f "$dockerfile" -t "$target" "$dockerdir"
+    docker build --progress plain -f "$dockerfile" -t "$target" "$dockerdir"
     docker push "$target"
 done

resources/apt_upgrade_images.sh

@@ -38,11 +38,20 @@ else
     cd "stac-browser"
 fi
 
+registry="ghcr.io/rs-python/stac-browser"
+
+# Set labels based on the Open Containers Initiative (OCI):
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+cat << EOF >> "Dockerfile"
+LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server"
+LABEL org.opencontainers.image.ref.name="$registry"
+LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/resources/deploy_stac_browser.sh"
+EOF
+
 # Docker image tag = 'last commit hash'.'last commit date'
 tag="$(git rev-parse --short HEAD).$(git log -1 --format="%at" | xargs -I{} date -d @{} +%Y-%m-%d)"
 
 # Build Docker image with tag + latest. It will be pushed to the rs-server registry.
-registry="ghcr.io/rs-python/stac-browser"
 docker build -t "${registry}:latest" -t "${registry}:${tag}" .
 
 # Push the images

resources/deploy_stac_browser.sh

@@ -64,3 +64,10 @@ USER user
 ENTRYPOINT [ \
     "python", "-m", "uvicorn", "rs_server_adgs.fastapi.adgs_app:app", \
     "--host", "0.0.0.0", "--port", "8000", "[DEBUG_MODE_RELOAD]" ]
+
+# Set labels based on the Open Containers Initiative (OCI):
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+#
+LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server"
+LABEL org.opencontainers.image.ref.name="ghcr.io/rs-python/rs-server-adgs"
+LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/services/adgs/.github/Dockerfile"