feat(docker): Add container image containing the CLP package, for Docker Compose integration (resolves #1164).

.github/actions/clp-execution-image-build/action.yaml → .github/actions/clp-image-build/action.yaml

@@ -1,8 +1,14 @@
-name: "clp-execution-image-build"
-description: "Builds a container image that contains the dependencies necessary
-to run the CLP package."
+name: "clp-image-build"
+description: "Builds a CLP container image."
 
 inputs:
+  image_type:
+    description: "Type of image to build"
+    required: true
+    type: "choice"
+    options:
+      - "execution"
+      - "package"
   image_registry:
     default: "ghcr.io"
     description: "Container image registry"
@@ -50,15 +56,29 @@ runs:
       uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
       with:
         images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\
-          /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}"
+          /clp-${{inputs.image_type}}-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}"
+
+    - name: "Determine Dockerfile Path"
+      id: "dockerfile"
+      shell: "bash"
+      run: |
+        base_path="./tools/docker-images"
+        platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}"
+
+        if [[ "${{inputs.image_type}}" == "execution" ]]; then
+          path="$base_path/clp-execution-base-$platform/Dockerfile"
+        else
+          path="$base_path/clp-package-$platform/Dockerfile"
+        fi
+
+        echo "DOCKERFILE=$path" >> "$GITHUB_OUTPUT"
 
     - name: "Build and Push"
       if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'"
       uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
       with:
         context: "./"
-        file: "./tools/docker-images/\
-          clp-execution-base-${{inputs.platform_id}}-${{inputs.platform_version_id}}/Dockerfile"
+        file: "${{steps.dockerfile.outputs.DOCKERFILE}}"
         push: true
         tags: "${{steps.meta.outputs.tags}}"
         labels: "${{steps.meta.outputs.labels}}"

.github/actions/run-on-image/action.yaml

@@ -45,6 +45,7 @@ runs:
     - run: >-
         docker run
         --user $(id -u):$(id -g)
+        --env npm_config_cache=/tmp/.npm
         --volume "$GITHUB_WORKSPACE":/mnt/repo
         --workdir /mnt/repo
         ${{steps.get_image_props.outputs.qualified_image_name}}

.github/workflows/clp-execution-image-build.yaml

@@ -3,12 +3,12 @@ name: "clp-execution-image-build"
 on:
   pull_request:
     paths:
-      - ".github/actions/clp-execution-image-build/action.yaml"
+      - ".github/actions/clp-image-build/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
   push:
     paths:
-      - ".github/actions/clp-execution-image-build/action.yaml"
+      - ".github/actions/clp-image-build/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
   schedule:
@@ -61,8 +61,9 @@ jobs:
         shell: "bash"
         run: "chown $(id -u):$(id -g) -R ."
 
-      - uses: "./.github/actions/clp-execution-image-build"
+      - uses: "./.github/actions/clp-image-build"
         with:
+          image_type: "execution"
           image_registry: "ghcr.io"
           image_registry_username: "${{github.actor}}"
           image_registry_password: "${{secrets.GITHUB_TOKEN}}"

.github/workflows/clp-package-image-build.yaml

@@ -0,0 +1,103 @@
+name: "clp-package-image-build"
+
+on:
+  push:
+  workflow_dispatch:
+
+env:
+  DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-"
+
+jobs:
+  filter-relevant-changes:
+    name: "filter-relevant-changes"
+    runs-on: "ubuntu-24.04"
+    outputs:
+      ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Work around actions/runner-images/issues/6775"
+        run: "chown $(id -u):$(id -g) -R ."
+        shell: "bash"
+
+      - name: "Filter relevant changes"
+        uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36"
+        id: "filter"
+        with:
+          # Consider changes between the current commit and `main`
+          # NOTE: If a pull request changes one of the images, then we need to (1) build the image
+          # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull
+          # request doesn't change an image, then we don't need to rebuild the image; instead we can
+          # use the published image which is based on `main`. So when determining what files have
+          # changed, we need to consider the delta between the current commit and `main` (rather
+          # than the current and previous commits) in order to detect if we need to rebuild the
+          # image (since it would be different from the published image).
+          base: "main"
+          filters: |
+            ubuntu_jammy_image:
+              - ".github/actions/**"
+              - ".github/workflows/clp-core-build.yaml"
+              - "components/core/tools/scripts/lib_install/*.sh"
+              - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**"
+              - "components/core/tools/scripts/lib_install/ubuntu-jammy/**"
+
+  ubuntu-jammy-deps-image:
+    name: "ubuntu-jammy-deps-image"
+    needs: "filter-relevant-changes"
+    runs-on: "ubuntu-24.04"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Work around actions/runner-images/issues/6775"
+        run: "chown $(id -u):$(id -g) -R ."
+        shell: "bash"
+
+      - uses: "./.github/actions/clp-core-build-containers"
+        env:
+          OS_NAME: "ubuntu-jammy"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          docker_context: "components/core"
+          docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\
+            /Dockerfile"
+          push_deps_image: >-
+            ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}}
+          token: "${{secrets.GITHUB_TOKEN}}"
+
+  ubuntu-jammy-package-image:
+    name: "ubuntu-jammy-package-image"
+    needs: "ubuntu-jammy-deps-image"
+    runs-on: "ubuntu-24.04"
+    env:
+      OS_NAME: "ubuntu-jammy"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Workaround actions/runner-images/issues/6775"
+        shell: "bash"
+        run: "chown $(id -u):$(id -g) -R ."
+
+      - name: "Build the package"
+        uses: "./.github/actions/run-on-image"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_published_image: >-
+            ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
+            || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
+          run_command: >-
+            CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package
+
+      - uses: "./.github/actions/clp-image-build"
+        with:
+          image_type: "package"
+          image_registry: "ghcr.io"
+          image_registry_username: "${{github.actor}}"
+          image_registry_password: "${{secrets.GITHUB_TOKEN}}"
+          platform_id: "ubuntu"
+          platform_version_id: "jammy"

components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh

@@ -23,8 +23,10 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
   openjdk-11-jdk \
   pkg-config \
   python3 \
+  python3-dev \
   python3-pip \
   python3-venv \
+  rsync \
   software-properties-common \
   unzip
 

docs/src/dev-guide/building-package.md

@@ -69,5 +69,16 @@ To clean up all build artifacts, run:
 task clean
 ```
 
+## Building a Docker image
+
+To build a Docker image containing the CLP package, run:
+
+```shell
+task docker-image-package
+```
+
+This will create a Docker image named `clp-package-<os>-ubuntu-jammy:dev` where `<os>` is `x86` or
+`arm64`.
+
 [clp-issue-872]: https://github.com/y-scope/clp/issues/872
 [Task]: https://taskfile.dev/

docs/src/dev-guide/tooling-containers.md

@@ -133,6 +133,16 @@ environment.
   tools/docker-images/clp-execution-base-ubuntu-jammy
   ```
 
+## clp-package-x86-ubuntu-jammy
+
+An image containing the CLP package built in an Ubuntu Jammy x86_64 environment.
+
+* Path:
+
+  ```text
+  tools/docker-images/clp-package-ubuntu-jammy
+  ```
+
 [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9
 [core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy
 [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy

docs/src/dev-guide/tooling-gh-workflows.md

@@ -76,6 +76,10 @@ This workflow builds CLP-core on macOS and runs its unit tests.
 This workflow builds a container image that contains the dependencies necessary to run the CLP
 package.
 
+## clp-package-image-build
+
+This workflow builds a container image that contains the CLP package.
+
 ## clp-lint
 
 This workflow runs linting checks on the codebase.

taskfile.yaml

@@ -162,6 +162,17 @@ tasks:
           CHECKSUM_FILE: "{{.CHECKSUM_FILE}}"
           INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"]
 
+  docker-image-package:
+    vars:
+      SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package-ubuntu-jammy"
+    dir: "{{.SRC_DIR}}"
+    deps:
+      - "package"
+    sources:
+      - "{{.SRC_DIR}}/**/*"
+    cmds:
+      - "./build.sh"
+
   clp-s-generate-parsers:
     vars:
       CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"

tools/docker-images/clp-package-ubuntu-jammy/Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:jammy AS base
+
+COPY ./build/clp-package /opt/clp
+
+WORKDIR /root
+
+COPY ./tools/docker-images/clp-package-ubuntu-jammy/setup-scripts ./setup-scripts
+RUN ./setup-scripts/install-prebuilt-packages.sh && \
+    rm -rf ./setup-scripts/
+
+# Remove cached files
+RUN apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Flatten the image
+FROM scratch
+COPY --from=base / /
+
+ENV PATH="/opt/clp/sbin:${PATH}"
+ENV PATH="/opt/clp/bin:${PATH}"
+ENV PYTHONPATH="/opt/clp/lib/python3/site-packages"
+ENV CLP_HOME="/opt/clp"

tools/docker-images/clp-package-ubuntu-jammy/build.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Exit on any error
+set -e
+
+# Error on undefined variable
+set -u
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+repo_root=${script_dir}/../../../
+
+arch=$(uname -m)
+
+if [ "$arch" = "x86_64" ]; then
+    arch_name="x86"
+elif [ "$arch" = "aarch64" ]; then
+    arch_name="arm64"
+else
+    echo "Error: Unsupported architecture - $arch"
+    exit 1
+fi
+
+build_cmd=(
+    docker build
+    --tag "clp-package-${arch_name}-ubuntu-jammy:dev"
+    "$repo_root"
+    --file "${script_dir}/Dockerfile"
+)
+
+if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ;
+then
+    build_cmd+=(
+        --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)"
+        --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)"
+    )
+fi
+
+"${build_cmd[@]}"

tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+# Exit on any error
+set -e
+
+# Error on undefined variable
+set -u
+
+apt-get update
+DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+  ca-certificates \
+  checkinstall \
+  curl \
+  libcurl4 \
+  libmariadb-dev \
+  libssl-dev \
+  python3 \
+  rsync \
+  zstd