feat(clp-package): Replace package execution container with clp-package; Change default taskfile task to docker-images:package. (#1357)

junhaoliao · web-flow · commit aba7ec4e5174 · 2025-10-01T21:04:54.000-04:00
diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml
@@ -2,9 +2,6 @@ name: "clp-build-runtime-image"
 description: "Builds a container image to be used for running CLP."
 
 inputs:
-  image_type:
-    description: "Type of image to build"
-    required: true
   image_registry:
     default: "ghcr.io"
     description: "Container image registry"
@@ -52,15 +49,8 @@ runs:
       shell: "bash"
       run: |
         base_path="./tools/docker-images"
-
-        if [[ "${{inputs.image_type}}" == "execution" ]]; then
-          platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}"
-          dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile"
-          image_name="clp-${{inputs.image_type}}-$platform"
-        else
-          dockerfile_path="$base_path/clp-package/Dockerfile"
-          image_name="clp-${{inputs.image_type}}"
-        fi
+        image_name="clp-package"
+        dockerfile_path="$base_path/$image_name/Dockerfile"
 
         echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT"
         echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml
@@ -541,7 +541,6 @@ jobs:
 
       - uses: "./.github/actions/clp-build-runtime-image"
         with:
-          image_type: "package"
           image_registry: "ghcr.io"
           image_registry_username: "${{github.actor}}"
           image_registry_password: "${{secrets.GITHUB_TOKEN}}"
diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml
diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py
@@ -427,7 +427,7 @@ def load_config_file(
         clp_config = CLPConfig()
 
     clp_config.make_config_paths_absolute(clp_home)
-    clp_config.load_execution_container_name()
+    clp_config.load_container_image_ref()
 
     validate_path_for_container_mount(clp_config.data_directory)
     validate_path_for_container_mount(clp_config.logs_directory)
diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py
@@ -229,7 +229,7 @@ def main(argv: List[str]) -> int:
         CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
     }
     container_start_cmd: List[str] = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
+        container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars
     )
 
     # fmt: off
diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py
@@ -249,7 +249,7 @@ def main(argv):
         CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
     }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
+        container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars
     )
     compress_cmd = _generate_compress_cmd(
         parsed_args, dataset, generated_config_path_on_container, logs_list_path_on_container
diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py
@@ -149,7 +149,7 @@ def main(argv: List[str]) -> int:
         CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
     }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
+        container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars
     )
 
     if len(aws_env_vars) != 0:
diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py
@@ -132,7 +132,7 @@ def handle_extract_file_cmd(
         CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
     }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
+        container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars
     )
 
     # fmt: off
@@ -211,7 +211,7 @@ def handle_extract_stream_cmd(
         CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
     }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
+        container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars
     )
 
     # fmt: off
diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py
@@ -134,7 +134,7 @@ def main(argv):
         CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,
     }
     container_start_cmd = generate_container_start_cmd(
-        container_name, necessary_mounts, clp_config.execution_container, extra_env_vars
+        container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars
     )
 
     # fmt: off
diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py
@@ -261,7 +261,7 @@ def create_db_tables(
         mounts.generated_config_file,
     ]
     append_docker_options(container_start_cmd, necessary_mounts, env_vars)
-    container_start_cmd.append(clp_config.execution_container)
+    container_start_cmd.append(clp_config.container_image_ref)
 
     clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils"
     # fmt: off
@@ -306,7 +306,7 @@ def create_results_cache_indices(
     env_vars = [f"PYTHONPATH={clp_site_packages_dir}"]
     necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir]
     append_docker_options(container_start_cmd, necessary_mounts, env_vars)
-    container_start_cmd.append(clp_config.execution_container)
+    container_start_cmd.append(clp_config.container_image_ref)
 
     clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils"
     # fmt: off
@@ -627,7 +627,7 @@ def generic_start_scheduler(
     ):
         necessary_mounts.append(mounts.input_logs_dir)
     append_docker_options(container_start_cmd, necessary_mounts, env_vars)
-    container_start_cmd.append(clp_config.execution_container)
+    container_start_cmd.append(clp_config.container_image_ref)
 
     # fmt: off
     scheduler_cmd = [
@@ -762,7 +762,7 @@ def generic_start_worker(
         env_vars.extend(aws_env_vars)
 
     append_docker_options(container_start_cmd, necessary_mounts, env_vars)
-    container_start_cmd.append(clp_config.execution_container)
+    container_start_cmd.append(clp_config.container_image_ref)
 
     worker_cmd = [
         "python3",
@@ -969,7 +969,7 @@ def start_webui(
         necessary_mounts.append(mounts.stream_output_dir)
 
     append_docker_options(container_cmd, necessary_mounts, env_vars)
-    container_cmd.append(clp_config.execution_container)
+    container_cmd.append(clp_config.container_image_ref)
 
     node_cmd = [
         str(CONTAINER_CLP_HOME / "bin" / "node-22"),
@@ -1023,7 +1023,7 @@ def start_reducer(
         mounts.generated_config_file,
     ]
     append_docker_options(container_start_cmd, necessary_mounts, env_vars)
-    container_start_cmd.append(clp_config.execution_container)
+    container_start_cmd.append(clp_config.container_image_ref)
 
     # fmt: off
     reducer_cmd = [
@@ -1101,7 +1101,7 @@ def start_garbage_collector(
         env_vars.extend(aws_env_vars)
 
     append_docker_options(container_start_cmd, necessary_mounts, env_vars)
-    container_start_cmd.append(clp_config.execution_container)
+    container_start_cmd.append(clp_config.container_image_ref)
 
     # fmt: off
     garbage_collector_cmd = [
diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py
@@ -83,14 +83,13 @@
 COMPRESSION_JOBS_TABLE_NAME = "compression_jobs"
 COMPRESSION_TASKS_TABLE_NAME = "compression_tasks"
 
-OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release"
-
 CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml"
 CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data"
 CLP_DEFAULT_DATASET_NAME = "default"
 CLP_METADATA_TABLE_PREFIX = "clp_"
+CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH = pathlib.Path("clp-package-image.id")
 CLP_SHARED_CONFIG_FILENAME = ".clp-config.yml"
-
+CLP_VERSION_FILE_PATH = pathlib.Path("VERSION")
 
 # Environment variable names
 CLP_DB_USER_ENV_VAR_NAME = "CLP_DB_USER"
@@ -876,7 +875,7 @@ def _get_env_var(name: str) -> str:
 
 
 class CLPConfig(BaseModel):
-    execution_container: Optional[str] = None
+    container_image_ref: Optional[str] = None
 
     logs_input: Union[FsIngestionConfig, S3IngestionConfig] = FsIngestionConfig()
 
@@ -902,7 +901,10 @@ class CLPConfig(BaseModel):
     logs_directory: pathlib.Path = pathlib.Path("var") / "log"
     aws_config_directory: Optional[pathlib.Path] = None
 
-    _os_release_file_path: pathlib.Path = PrivateAttr(default=OS_RELEASE_FILE_PATH)
+    _container_image_id_path: pathlib.Path = PrivateAttr(
+        default=CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH
+    )
+    _version_file_path: pathlib.Path = PrivateAttr(default=CLP_VERSION_FILE_PATH)
 
     @field_validator("aws_config_directory")
     @classmethod
@@ -919,7 +921,10 @@ def make_config_paths_absolute(self, clp_home: pathlib.Path):
         self.stream_output.storage.make_config_paths_absolute(clp_home)
         self.data_directory = make_config_path_absolute(clp_home, self.data_directory)
         self.logs_directory = make_config_path_absolute(clp_home, self.logs_directory)
-        self._os_release_file_path = make_config_path_absolute(clp_home, self._os_release_file_path)
+        self._container_image_id_path = make_config_path_absolute(
+            clp_home, self._container_image_id_path
+        )
+        self._version_file_path = make_config_path_absolute(clp_home, self._version_file_path)
 
     def validate_logs_input_config(self):
         logs_input_type = self.logs_input.type
@@ -1007,22 +1012,18 @@ def validate_aws_config_dir(self):
                 "aws_config_directory should not be set when profile authentication is not used"
             )
 
-    def load_execution_container_name(self):
-        if self.execution_container is not None:
+    def load_container_image_ref(self):
+        if self.container_image_ref is not None:
             # Accept configured value for debug purposes
             return
 
-        os_release = dotenv_values(self._os_release_file_path)
-        if "ubuntu" == os_release["ID"]:
-            self.execution_container = (
-                f"clp-execution-x86-{os_release['ID']}-{os_release['VERSION_CODENAME']}:main"
-            )
+        if self._container_image_id_path.exists():
+            with open(self._container_image_id_path) as image_id_file:
+                self.container_image_ref = image_id_file.read().strip()
         else:
-            raise NotImplementedError(
-                f"Unsupported OS {os_release['ID']} in {OS_RELEASE_FILE_PATH}"
-            )
-
-        self.execution_container = "ghcr.io/y-scope/clp/" + self.execution_container
+            with open(self._version_file_path) as version_file:
+                clp_package_version = version_file.read().strip()
+            self.container_image_ref = f"ghcr.io/y-scope/clp/clp-package:{clp_package_version}"
 
     def get_shared_config_file_path(self) -> pathlib.Path:
         return self.logs_directory / CLP_SHARED_CONFIG_FILENAME
diff --git a/docs/src/dev-docs/tooling-containers.md b/docs/src/dev-docs/tooling-containers.md
@@ -129,24 +129,6 @@ Jammy x86 environment.
   components/core/tools/docker-images/clp-core-ubuntu-jammy
   ```
 
-## clp-execution-x86-ubuntu-jammy
-
-An image containing the dependencies necessary to run the CLP package in an Ubuntu Jammy x86
-environment.
-
-* [GitHub Packages page][exe-ubuntu-jammy]
-* Pull command:
-
-  ```bash
-  docker pull ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main
-  ```
-
-* Path:
-
-  ```text
-  tools/docker-images/clp-execution-base-ubuntu-jammy
-  ```
-
 ## clp-package
 
 An image containing the CLP package.
@@ -162,7 +144,6 @@ An image containing the CLP package.
 [core-deps-musllinux_1_2-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-musllinux_1_2
 [core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy
 [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy
-[exe-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy
 [gh-packages]: https://github.com/orgs/y-scope/packages?repo_name=clp
 [manylinux_2_28]: https://github.com/pypa/manylinux?tab=readme-ov-file#manylinux_2_28-almalinux-8-based
 [musllinux_1_2]: https://github.com/pypa/manylinux?tab=readme-ov-file#musllinux_1_2-alpine-linux-322-based-313-compatible
diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md
@@ -88,11 +88,6 @@ need it for one OS since users can use the container on any OS.
 
 This workflow builds CLP-core on macOS and runs its unit tests.
 
-## clp-execution-image-build
-
-This workflow builds a container image that contains the dependencies necessary to run the CLP
-package.
-
 ## clp-lint
 
 This workflow runs linting checks on the codebase.
diff --git a/taskfile.yaml b/taskfile.yaml
@@ -52,7 +52,7 @@ vars:
 
 tasks:
   default:
-    deps: ["package"]
+    deps: ["docker-images:package"]
 
   clean:
     cmds:
@@ -113,7 +113,6 @@ tasks:
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/indexer"
       - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server"
       - "{{.TASKFILE}}"
-      - "/etc/os-release"
       - "components/clp-package-utils/dist/*.whl"
       - "components/clp-py-utils/dist/*.whl"
       - "components/job-orchestration/dist/*.whl"
@@ -135,7 +134,6 @@ tasks:
     cmds:
       - "rm -rf '{{.OUTPUT_DIR}}'"
       - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'"
-      - "rsync --copy-links /etc/os-release '{{.OUTPUT_DIR}}/etc/os-release'"
       - "mkdir -p '{{.OUTPUT_DIR}}/lib/python3/site-packages'"
       - |-
         . "{{.G_PACKAGE_VENV_DIR}}/bin/activate"
diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml
@@ -12,3 +12,4 @@ tasks:
       - ":package"
     cmds:
       - "./build.sh"
+      - "rsync --archive '{{.G_BUILD_DIR}}/clp-package-image.id' '{{.G_PACKAGE_BUILD_DIR}}'"
diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile
diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh
diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh

Original file line number	Diff line number	Diff line change
`@@ -229,7 +229,7 @@ def main(argv: List[str]) -> int:`
`229`	`229`	`CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,`
`230`	`230`	`}`
`231`	`231`	`container_start_cmd: List[str] = generate_container_start_cmd(`
`232`		`- container_name, necessary_mounts, clp_config.execution_container, extra_env_vars`
	`232`	`+ container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars`
`233`	`233`	`)`
`234`	`234`
`235`	`235`	`# fmt: off`
Original file line number	Diff line number	Diff line change
`@@ -249,7 +249,7 @@ def main(argv):`
`249`	`249`	`CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,`
`250`	`250`	`}`
`251`	`251`	`container_start_cmd = generate_container_start_cmd(`
`252`		`- container_name, necessary_mounts, clp_config.execution_container, extra_env_vars`
	`252`	`+ container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars`
`253`	`253`	`)`
`254`	`254`	`compress_cmd = _generate_compress_cmd(`
`255`	`255`	`parsed_args, dataset, generated_config_path_on_container, logs_list_path_on_container`
Original file line number	Diff line number	Diff line change
`@@ -149,7 +149,7 @@ def main(argv: List[str]) -> int:`
`149`	`149`	`CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,`
`150`	`150`	`}`
`151`	`151`	`container_start_cmd = generate_container_start_cmd(`
`152`		`- container_name, necessary_mounts, clp_config.execution_container, extra_env_vars`
	`152`	`+ container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars`
`153`	`153`	`)`
`154`	`154`
`155`	`155`	`if len(aws_env_vars) != 0:`
Original file line number	Diff line number	Diff line change
`@@ -132,7 +132,7 @@ def handle_extract_file_cmd(`
`132`	`132`	`CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,`
`133`	`133`	`}`
`134`	`134`	`container_start_cmd = generate_container_start_cmd(`
`135`		`- container_name, necessary_mounts, clp_config.execution_container, extra_env_vars`
	`135`	`+ container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars`
`136`	`136`	`)`
`137`	`137`
`138`	`138`	`# fmt: off`
`@@ -211,7 +211,7 @@ def handle_extract_stream_cmd(`
`211`	`211`	`CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,`
`212`	`212`	`}`
`213`	`213`	`container_start_cmd = generate_container_start_cmd(`
`214`		`- container_name, necessary_mounts, clp_config.execution_container, extra_env_vars`
	`214`	`+ container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars`
`215`	`215`	`)`
`216`	`216`
`217`	`217`	`# fmt: off`
Original file line number	Diff line number	Diff line change
`@@ -134,7 +134,7 @@ def main(argv):`
`134`	`134`	`CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password,`
`135`	`135`	`}`
`136`	`136`	`container_start_cmd = generate_container_start_cmd(`
`137`		`- container_name, necessary_mounts, clp_config.execution_container, extra_env_vars`
	`137`	`+ container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars`
`138`	`138`	`)`
`139`	`139`
`140`	`140`	`# fmt: off`