feat(docker): Add container image containing the CLP package, for Docker Compose integration (resolves #1164).

.github/actions/clp-execution-image-build/action.yaml → .github/actions/clp-build-runtime-image/action.yaml

@@ -1,8 +1,10 @@
-name: "clp-execution-image-build"
-description: "Builds a container image that contains the dependencies necessary
-to run the CLP package."
+name: "clp-build-runtime-image"
+description: "Builds a container image to be used for running CLP."
 
 inputs:
+  image_type:
+    description: "Type of image to build"
+    required: true
   image_registry:
     default: "ghcr.io"
     description: "Container image registry"
@@ -17,11 +19,11 @@ inputs:
     required: false
   platform_id:
     description: "Platform ID of the container (e.g. ubuntu)"
-    required: true
+    required: false
   platform_version_id:
     description: "Platform VERSION_ID / VERSION_CODENAME of the container
       (e.g. jammy, focal, etc.)"
-    required: true
+    required: false
 
 runs:
   using: "composite"
@@ -45,20 +47,37 @@ runs:
         echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \
           >> "$GITHUB_OUTPUT"
 
-    - name: "Update Metadata"
-      id: "meta"
+    - name: "Compute metadata"
+      id: "compute-meta"
+      shell: "bash"
+      run: |
+        base_path="./tools/docker-images"
+
+        if [[ "${{inputs.image_type}}" == "execution" ]]; then
+          platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}"
+          dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile"
+          image_name="clp-${{inputs.image_type}}-$platform"
+        else
+          dockerfile_path="$base_path/clp-package/Dockerfile"
+          image_name="clp-${{inputs.image_type}}"
+        fi
+
+        echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT"
+        echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT"
+
+    - name: "Extract GitHub Metadata"
+      id: "extract-gh-meta"
       uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
       with:
         images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\
-          /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}"
+          /${{steps.compute-meta.outputs.IMAGE_NAME}}"
 
     - name: "Build and Push"
       if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'"
       uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
       with:
         context: "./"
-        file: "./tools/docker-images/\
-          clp-execution-base-${{inputs.platform_id}}-${{inputs.platform_version_id}}/Dockerfile"
+        file: "${{steps.compute-meta.outputs.DOCKERFILE}}"
         push: true
-        tags: "${{steps.meta.outputs.tags}}"
-        labels: "${{steps.meta.outputs.labels}}"
+        tags: "${{steps.extract-gh-meta.outputs.tags}}"
+        labels: "${{steps.extract-gh-meta.outputs.labels}}"

.github/actions/run-on-image/action.yaml

@@ -42,9 +42,11 @@ runs:
     - run: "./tools/scripts/deps-download/init.sh"
       shell: "bash"
 
+    # `--env npm_config_cache` overrides the default `/.npm` to avoid permission issues.
     - run: >-
         docker run
         --user $(id -u):$(id -g)
+        --env npm_config_cache=/tmp/.npm
         --volume "$GITHUB_WORKSPACE":/mnt/repo
         --workdir /mnt/repo
         ${{steps.get_image_props.outputs.qualified_image_name}}

.github/workflows/clp-core-build.yaml

@@ -2,27 +2,13 @@ name: "clp-core-build"
 
 on:
   pull_request:
-    paths:
-      - ".github/actions/clp-core-build-containers/action.yaml"
-      - ".github/actions/run-on-image/action.yaml"
-      - ".github/workflows/clp-core-build.yaml"
-      - ".gitmodules"
-      - "components/core/**"
-      - "taskfile.yaml"
-      - "taskfiles/**"
-      - "tools/scripts/deps-download/**"
-      - "!components/core/tools/scripts/lib_install/macos/**"
+    paths-ignore: &ignored_paths
+      - ".github/*"
+      - ".github/ISSUE_TEMPLATE/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
+      - "docs/**"
   push:
-    paths:
-      - ".github/actions/clp-core-build-containers/action.yaml"
-      - ".github/actions/run-on-image/action.yaml"
-      - ".github/workflows/clp-core-build.yaml"
-      - ".gitmodules"
-      - "components/core/**"
-      - "taskfile.yaml"
-      - "taskfiles/**"
-      - "tools/scripts/deps-download/**"
-      - "!components/core/tools/scripts/lib_install/macos/**"
+    paths-ignore: *ignored_paths
   schedule:
     # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
     - cron: "15 0 * * *"
@@ -520,3 +506,42 @@ jobs:
             .task/checksum/utils-cpp-lint-clang-tidy-*
             build/lint-clang-tidy
           key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}"
+
+  package-image:
+    name: "package-image"
+    if: >-
+      !cancelled() && !failure() && (
+        needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' ||
+        needs.ubuntu-jammy-deps-image.result == 'success'
+      )
+    needs:
+      - "filter-relevant-changes"
+      - "ubuntu-jammy-deps-image"
+    runs-on: "ubuntu-24.04"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Workaround actions/runner-images/issues/6775"
+        shell: "bash"
+        run: "chown $(id -u):$(id -g) -R ."
+
+      - name: "Build the package"
+        uses: "./.github/actions/run-on-image"
+        env:
+          OS_NAME: "ubuntu-jammy"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_published_image: >-
+            ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
+            || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
+          run_command: >-
+            CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package
+
+      - uses: "./.github/actions/clp-build-runtime-image"
+        with:
+          image_type: "package"
+          image_registry: "ghcr.io"
+          image_registry_username: "${{github.actor}}"
+          image_registry_password: "${{secrets.GITHUB_TOKEN}}"

.github/workflows/clp-execution-image-build.yaml

@@ -3,12 +3,12 @@ name: "clp-execution-image-build"
 on:
   pull_request:
     paths:
-      - ".github/actions/clp-execution-image-build/action.yaml"
+      - ".github/actions/clp-build-runtime-image/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
   push:
     paths:
-      - ".github/actions/clp-execution-image-build/action.yaml"
+      - ".github/actions/clp-build-runtime-image/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
   schedule:
@@ -61,8 +61,9 @@ jobs:
         shell: "bash"
         run: "chown $(id -u):$(id -g) -R ."
 
-      - uses: "./.github/actions/clp-execution-image-build"
+      - uses: "./.github/actions/clp-build-runtime-image"
         with:
+          image_type: "execution"
           image_registry: "ghcr.io"
           image_registry_username: "${{github.actor}}"
           image_registry_password: "${{secrets.GITHUB_TOKEN}}"

components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh

@@ -23,8 +23,10 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
   openjdk-11-jdk \
   pkg-config \
   python3 \
+  python3-dev \
   python3-pip \
   python3-venv \
+  rsync \
   software-properties-common \
   unzip
 

docs/src/dev-docs/building-package.md

@@ -69,5 +69,15 @@ To clean up all build artifacts, run:
 task clean
 ```
 
+## Building a Docker image
+
+To build a Docker image containing the CLP package, run:
+
+```shell
+task docker-images:package
+```
+
+This will create a Docker image named `clp-package:dev`.
+
 [clp-issue-872]: https://github.com/y-scope/clp/issues/872
 [Task]: https://taskfile.dev/

docs/src/dev-docs/tooling-containers.md

@@ -147,6 +147,16 @@ environment.
   tools/docker-images/clp-execution-base-ubuntu-jammy
   ```
 
+## clp-package
+
+An image containing the CLP package.
+
+* Path:
+
+  ```text
+  tools/docker-images/clp-package
+  ```
+
 [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9
 [core-deps-manylinux_2_28-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-manylinux_2_28
 [core-deps-musllinux_1_2-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-musllinux_1_2

docs/src/dev-docs/tooling-gh-workflows.md

@@ -90,6 +90,10 @@ This workflow builds CLP-core on macOS and runs its unit tests.
 This workflow builds a container image that contains the dependencies necessary to run the CLP
 package.
 
+## clp-package-image-build
+
+This workflow builds a container image that contains the CLP package.
+
 ## clp-lint
 
 This workflow runs linting checks on the codebase.

taskfile.yaml

@@ -5,6 +5,7 @@ shopt: ["globstar"]
 
 includes:
   deps: "taskfiles/deps/main.yaml"
+  docker-images: "taskfiles/docker-images.yaml"
   docs: "taskfiles/docs.yaml"
   lint: "taskfiles/lint.yaml"
   utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml"

taskfiles/docker-images.yaml

@@ -0,0 +1,13 @@
+version: "3"
+
+tasks:
+  package:
+    vars:
+      SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package"
+    dir: "{{.SRC_DIR}}"
+    sources:
+      - "{{.SRC_DIR}}/**/*"
+    deps:
+      - ":package"
+    cmds:
+      - "./build.sh"

tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh

@@ -9,11 +9,6 @@ set -u
 apt-get update
 DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
   ca-certificates \
-  checkinstall \
-  curl \
   libcurl4 \
-  libmariadb-dev \
-  libssl-dev \
-  python3 \
-  rsync \
-  zstd
+  libmariadb3 \
+  python3

tools/docker-images/clp-package/Dockerfile

@@ -0,0 +1,24 @@
+FROM ubuntu:jammy AS base
+
+COPY ./build/clp-package /opt/clp
+
+WORKDIR /root
+
+COPY ./tools/docker-images/clp-package/setup-scripts ./setup-scripts
+RUN ./setup-scripts/install-prebuilt-packages.sh \
+    && rm -rf ./setup-scripts/
+
+# Remove cached files
+RUN apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Flatten the image
+FROM scratch
+COPY --from=base / /
+
+ENV CLP_HOME="/opt/clp"
+ENV PATH="${CLP_HOME}/bin:${PATH}"
+ENV PATH="${CLP_HOME}/sbin:${PATH}"
+ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages"
+
+USER 1000:1000

tools/docker-images/clp-package/build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+repo_root=${script_dir}/../../../
+
+build_cmd=(
+    docker build
+    --tag "clp-package:dev"
+    "$repo_root"
+    --file "${script_dir}/Dockerfile"
+)
+
+if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ;
+then
+    build_cmd+=(
+        --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
+        --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
+    )
+fi
+
+"${build_cmd[@]}"

tools/docker-images/clp-package/setup-scripts/install-prebuilt-packages.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -eu
+set -o pipefail
+
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+  ca-certificates \
+  libcurl4 \
+  libmariadb3 \
+  python3