feat(docker): Add container image containing the CLP package, for Docker Compose integration (resolves #1164). (#1166)

Co-authored-by: kirkrodrigues <[email protected]>

Author: junhaoliao

.github/actions/clp-execution-image-build/action.yaml renamed to .github/actions/clp-build-runtime-image/action.yaml

@@ -1,8 +1,10 @@
-name: "clp-execution-image-build"
-description: "Builds a container image that contains the dependencies necessary
-to run the CLP package."
+name: "clp-build-runtime-image"
+description: "Builds a container image to be used for running CLP."
 
 inputs:
+  image_type:
+    description: "Type of image to build"
+    required: true
   image_registry:
     default: "ghcr.io"
     description: "Container image registry"
@@ -17,11 +19,11 @@ inputs:
     required: false
   platform_id:
     description: "Platform ID of the container (e.g. ubuntu)"
-    required: true
+    required: false
   platform_version_id:
     description: "Platform VERSION_ID / VERSION_CODENAME of the container
       (e.g. jammy, focal, etc.)"
-    required: true
+    required: false
 
 runs:
   using: "composite"
@@ -45,20 +47,37 @@ runs:
         echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \
           >> "$GITHUB_OUTPUT"
 
-    - name: "Update Metadata"
-      id: "meta"
+    - name: "Compute metadata"
+      id: "compute-meta"
+      shell: "bash"
+      run: |
+        base_path="./tools/docker-images"
+
+        if [[ "${{inputs.image_type}}" == "execution" ]]; then
+          platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}"
+          dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile"
+          image_name="clp-${{inputs.image_type}}-$platform"
+        else
+          dockerfile_path="$base_path/clp-package/Dockerfile"
+          image_name="clp-${{inputs.image_type}}"
+        fi
+
+        echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT"
+        echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT"
+
+    - name: "Extract GitHub Metadata"
+      id: "extract-gh-meta"
       uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804"
       with:
         images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\
-          /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}"
+          /${{steps.compute-meta.outputs.IMAGE_NAME}}"
 
     - name: "Build and Push"
       if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'"
       uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4"
       with:
         context: "./"
-        file: "./tools/docker-images/\
-          clp-execution-base-${{inputs.platform_id}}-${{inputs.platform_version_id}}/Dockerfile"
+        file: "${{steps.compute-meta.outputs.DOCKERFILE}}"
         push: true
-        tags: "${{steps.meta.outputs.tags}}"
-        labels: "${{steps.meta.outputs.labels}}"
+        tags: "${{steps.extract-gh-meta.outputs.tags}}"
+        labels: "${{steps.extract-gh-meta.outputs.labels}}"

.github/actions/run-on-image/action.yaml

@@ -42,9 +42,11 @@ runs:
     - run: "./tools/scripts/deps-download/init.sh"
       shell: "bash"
 
+    # `--env npm_config_cache` overrides the default `/.npm` to avoid permission issues.
     - run: >-
         docker run
         --user $(id -u):$(id -g)
+        --env npm_config_cache=/tmp/.npm
         --volume "$GITHUB_WORKSPACE":/mnt/repo
         --workdir /mnt/repo
         ${{steps.get_image_props.outputs.qualified_image_name}}

.github/workflows/clp-core-build.yaml renamed to .github/workflows/clp-artifact-build.yaml

@@ -1,28 +1,14 @@
-name: "clp-core-build"
+name: "clp-artifact-build"
 
 on:
   pull_request:
-    paths:
-      - ".github/actions/clp-core-build-containers/action.yaml"
-      - ".github/actions/run-on-image/action.yaml"
-      - ".github/workflows/clp-core-build.yaml"
-      - ".gitmodules"
-      - "components/core/**"
-      - "taskfile.yaml"
-      - "taskfiles/**"
-      - "tools/scripts/deps-download/**"
-      - "!components/core/tools/scripts/lib_install/macos/**"
+    paths-ignore: &ignored_paths
+      - ".github/*"
+      - ".github/ISSUE_TEMPLATE/**"
+      - "components/core/tools/scripts/lib_install/macos/**"
+      - "docs/**"
   push:
-    paths:
-      - ".github/actions/clp-core-build-containers/action.yaml"
-      - ".github/actions/run-on-image/action.yaml"
-      - ".github/workflows/clp-core-build.yaml"
-      - ".gitmodules"
-      - "components/core/**"
-      - "taskfile.yaml"
-      - "taskfiles/**"
-      - "tools/scripts/deps-download/**"
-      - "!components/core/tools/scripts/lib_install/macos/**"
+    paths-ignore: *ignored_paths
   schedule:
     # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
     - cron: "15 0 * * *"
@@ -74,31 +60,31 @@ jobs:
           filters: |
             centos_stream_9_image:
               - ".github/actions/**"
-              - ".github/workflows/clp-core-build.yaml"
+              - ".github/workflows/clp-artifact-build.yaml"
               - "components/core/tools/scripts/lib_install/*.sh"
               - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**"
               - "components/core/tools/scripts/lib_install/centos-stream-9/**"
             manylinux_2_28_x86_64_image:
               - ".github/actions/**"
-              - ".github/workflows/clp-core-build.yaml"
+              - ".github/workflows/clp-artifact-build.yaml"
               - "components/core/tools/scripts/lib_install/*.sh"
               - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**"
               - "components/core/tools/scripts/lib_install/manylinux_2_28/**"
             musllinux_1_2_x86_64_image:
               - ".github/actions/**"
-              - ".github/workflows/clp-core-build.yaml"
+              - ".github/workflows/clp-artifact-build.yaml"
               - "components/core/tools/scripts/lib_install/*.sh"
               - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/**"
               - "components/core/tools/scripts/lib_install/musllinux_1_2/**"
             ubuntu_jammy_image:
               - ".github/actions/**"
-              - ".github/workflows/clp-core-build.yaml"
+              - ".github/workflows/clp-artifact-build.yaml"
               - "components/core/tools/scripts/lib_install/*.sh"
               - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**"
               - "components/core/tools/scripts/lib_install/ubuntu-jammy/**"
             clp:
               - ".github/actions/**"
-              - ".github/workflows/clp-core-build.yaml"
+              - ".github/workflows/clp-artifact-build.yaml"
               - ".gitmodules"
               - "components/core/cmake/**"
               - "components/core/CMakeLists.txt"
@@ -520,3 +506,42 @@ jobs:
             .task/checksum/utils-cpp-lint-clang-tidy-*
             build/lint-clang-tidy
           key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}"
+
+  package-image:
+    name: "package-image"
+    if: >-
+      !cancelled() && !failure() && (
+        needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' ||
+        needs.ubuntu-jammy-deps-image.result == 'success'
+      )
+    needs:
+      - "filter-relevant-changes"
+      - "ubuntu-jammy-deps-image"
+    runs-on: "ubuntu-24.04"
+    steps:
+      - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683"
+        with:
+          submodules: "recursive"
+
+      - name: "Workaround actions/runner-images/issues/6775"
+        shell: "bash"
+        run: "chown $(id -u):$(id -g) -R ."
+
+      - name: "Build the package"
+        uses: "./.github/actions/run-on-image"
+        env:
+          OS_NAME: "ubuntu-jammy"
+        with:
+          image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}"
+          use_published_image: >-
+            ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false'
+            || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}}
+          run_command: >-
+            CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package
+
+      - uses: "./.github/actions/clp-build-runtime-image"
+        with:
+          image_type: "package"
+          image_registry: "ghcr.io"
+          image_registry_username: "${{github.actor}}"
+          image_registry_password: "${{secrets.GITHUB_TOKEN}}"

.github/workflows/clp-execution-image-build.yaml

@@ -3,12 +3,12 @@ name: "clp-execution-image-build"
 on:
   pull_request:
     paths:
-      - ".github/actions/clp-execution-image-build/action.yaml"
+      - ".github/actions/clp-build-runtime-image/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
   push:
     paths:
-      - ".github/actions/clp-execution-image-build/action.yaml"
+      - ".github/actions/clp-build-runtime-image/action.yaml"
       - ".github/workflows/clp-execution-image-build.yaml"
       - "tools/docker-images/**/*"
   schedule:
@@ -61,8 +61,9 @@ jobs:
         shell: "bash"
         run: "chown $(id -u):$(id -g) -R ."
 
-      - uses: "./.github/actions/clp-execution-image-build"
+      - uses: "./.github/actions/clp-build-runtime-image"
         with:
+          image_type: "execution"
           image_registry: "ghcr.io"
           image_registry_username: "${{github.actor}}"
           image_registry_password: "${{secrets.GITHUB_TOKEN}}"

components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh

@@ -23,8 +23,10 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
   openjdk-11-jdk \
   pkg-config \
   python3 \
+  python3-dev \
   python3-pip \
   python3-venv \
+  rsync \
   software-properties-common \
   unzip
 

docs/src/dev-docs/building-package.md

@@ -68,5 +68,15 @@ To clean up all build artifacts, run:
 task clean
 ```
 
+## Building a Docker image
+
+To build a Docker image containing the CLP package, run:
+
+```shell
+task docker-images:package
+```
+
+This will create a Docker image named `clp-package:dev`.
+
 [clp-issue-872]: https://github.com/y-scope/clp/issues/872
 [Task]: https://taskfile.dev/

docs/src/dev-docs/contributing-linting.md

@@ -2,7 +2,7 @@
 
 Before submitting a PR, ensure you've run our linting tools and either fixed any violations or
 suppressed the warning. If you can't run the linting workflows locally, you can enable and run the
-[clp-lint] and [clp-core-build] workflows in your fork.
+[clp-lint] and [clp-artifact-build] workflows in your fork.
 
 ## Requirements
 
@@ -32,7 +32,7 @@ To also apply any automatic fixes:
 task lint:fix
 ```
 
-[clp-core-build]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-core-build.yaml
+[clp-artifact-build]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-artifact-build.yaml
 [clp-lint]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-lint.yaml
 [clp-issue-872]: https://github.com/y-scope/clp/issues/872
 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&projects=&template=feature-request.yml

docs/src/dev-docs/tooling-containers.md

@@ -147,6 +147,16 @@ environment.
   tools/docker-images/clp-execution-base-ubuntu-jammy
   ```
 
+## clp-package
+
+An image containing the CLP package.
+
+* Path:
+
+  ```text
+  tools/docker-images/clp-package
+  ```
+
 [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9
 [core-deps-manylinux_2_28-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-manylinux_2_28
 [core-deps-musllinux_1_2-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-musllinux_1_2

docs/src/dev-docs/tooling-gh-workflows.md

@@ -3,12 +3,13 @@
 The CLP repo includes several GitHub workflows for automating container image builds, artifact
 builds, testing, and linting. We briefly describe each workflow below.
 
-## clp-core-build
+## clp-artifact-build
 
 This workflow is responsible for:
 
-1. building (Linux) container images containing CLP-core's dependencies, and
-2. building CLP-core and running its unit tests.
+1. building (Linux) container images containing CLP-core's dependencies,
+2. building CLP-core and running its unit tests, and
+3. building a container image containing CLP's package components.
 
 To minimize build times, the jobs in the workflow are organized in the directed acyclic graph (DAG)
 shown below.
@@ -40,6 +41,7 @@ flowchart LR
     manylinux_2_28-x86_64-deps-image --> manylinux_2_28-x86_64-binaries
     musllinux_1_2-x86_64-deps-image --> musllinux_1_2-x86_64-binaries
     ubuntu-jammy-deps-image --> ubuntu-jammy-binaries
+    ubuntu-jammy-deps-image --> package-image
     ubuntu-jammy-binaries --> ubuntu-jammy-binaries-image
 :::
 
@@ -61,6 +63,7 @@ Arrows between jobs indicate a dependency. The jobs are as follows:
   container and runs core's unit tests.
 * `musllinux_1_2-x86_64-binaries`: Builds the CLP-core binaries in the built musllinux_1_2 container
   and runs core's unit tests.
+* `package-image`: Builds a container image containing CLP's package components.
 * `ubuntu-jammy-binaries`: Builds the CLP-core binaries in the built Ubuntu Jammy container and runs
   core's unit tests.
 * `ubuntu-jammy-binaries-image`: Builds an Ubuntu Jammy container image containing CLP-core's

taskfile.yaml

@@ -5,6 +5,7 @@ shopt: ["globstar"]
 
 includes:
   deps: "taskfiles/deps/main.yaml"
+  docker-images: "taskfiles/docker-images.yaml"
   docs: "taskfiles/docs.yaml"
   lint: "taskfiles/lint.yaml"
   utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml"
@@ -37,6 +38,7 @@ vars:
     {{default "" (env "CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK")}}
 
   # Checksum files
+  G_PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/package.md5"
   G_WEBUI_CLIENT_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-client-node-modules.md5"
   G_WEBUI_COMMON_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-common-node-modules.md5"
   G_WEBUI_LOG_VIEWER_NODE_MODULES_CHECKSUM_FILE:
@@ -94,7 +96,7 @@ tasks:
     env:
       NODE_ENV: "production"
     vars:
-      CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
+      CHECKSUM_FILE: "{{.G_PACKAGE_CHECKSUM_FILE}}"
       OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}"
     sources:
       - "{{.G_BUILD_DIR}}/package-venv.md5"