diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index ecf3153994..32e984bfa9 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -2,9 +2,6 @@ name: "clp-build-runtime-image" description: "Builds a container image to be used for running CLP." inputs: - image_type: - description: "Type of image to build" - required: true image_registry: default: "ghcr.io" description: "Container image registry" @@ -52,15 +49,8 @@ runs: shell: "bash" run: | base_path="./tools/docker-images" - - if [[ "${{inputs.image_type}}" == "execution" ]]; then - platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" - dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile" - image_name="clp-${{inputs.image_type}}-$platform" - else - dockerfile_path="$base_path/clp-package/Dockerfile" - image_name="clp-${{inputs.image_type}}" - fi + image_name="clp-package" + dockerfile_path="$base_path/$image_name/Dockerfile" echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index de4578aecc..59b216b43c 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -541,7 +541,6 @@ jobs: - uses: "./.github/actions/clp-build-runtime-image" with: - image_type: "package" image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" image_registry_password: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 9586bbbeb5..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-build-runtime-image/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-build-runtime-image/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-24.04" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-24.04" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-build-runtime-image" - with: - image_type: "execution" - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index cb0d708077..a09bb7ae50 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -427,7 +427,7 @@ def load_config_file( clp_config = CLPConfig() clp_config.make_config_paths_absolute(clp_home) - clp_config.load_execution_container_name() + clp_config.load_container_image_ref() validate_path_for_container_mount(clp_config.data_directory) validate_path_for_container_mount(clp_config.logs_directory) diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index ae9ab40d49..b965805d11 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -229,7 +229,7 @@ def main(argv: List[str]) -> int: CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, } container_start_cmd: List[str] = generate_container_start_cmd( - container_name, necessary_mounts, clp_config.execution_container, extra_env_vars + container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars ) # fmt: off diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 0b02a65c08..ae9de02092 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -249,7 +249,7 @@ def main(argv): CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, } container_start_cmd = generate_container_start_cmd( - container_name, necessary_mounts, clp_config.execution_container, extra_env_vars + container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars ) compress_cmd = _generate_compress_cmd( parsed_args, dataset, generated_config_path_on_container, logs_list_path_on_container diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 55c280e489..08c0cebe16 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -149,7 +149,7 @@ def main(argv: List[str]) -> int: CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, } container_start_cmd = generate_container_start_cmd( - container_name, necessary_mounts, clp_config.execution_container, extra_env_vars + container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars ) if len(aws_env_vars) != 0: diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index 90727e3408..7c4a1d5f62 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -132,7 +132,7 @@ def handle_extract_file_cmd( CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, } container_start_cmd = generate_container_start_cmd( - container_name, necessary_mounts, clp_config.execution_container, extra_env_vars + container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars ) # fmt: off @@ -211,7 +211,7 @@ def handle_extract_stream_cmd( CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, } container_start_cmd = generate_container_start_cmd( - container_name, necessary_mounts, clp_config.execution_container, extra_env_vars + container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars ) # fmt: off diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index eb5a2118b0..a69d97db6f 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -134,7 +134,7 @@ def main(argv): CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, } container_start_cmd = generate_container_start_cmd( - container_name, necessary_mounts, clp_config.execution_container, extra_env_vars + container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars ) # fmt: off diff --git a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py index addf8fb9dd..f41ff8ab7b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/start_clp.py +++ b/components/clp-package-utils/clp_package_utils/scripts/start_clp.py @@ -261,7 +261,7 @@ def create_db_tables( mounts.generated_config_file, ] append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) + container_start_cmd.append(clp_config.container_image_ref) clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils" # fmt: off @@ -306,7 +306,7 @@ def create_results_cache_indices( env_vars = [f"PYTHONPATH={clp_site_packages_dir}"] necessary_mounts = [mounts.clp_home, mounts.data_dir, mounts.logs_dir] append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) + container_start_cmd.append(clp_config.container_image_ref) clp_py_utils_dir = clp_site_packages_dir / "clp_py_utils" # fmt: off @@ -627,7 +627,7 @@ def generic_start_scheduler( ): necessary_mounts.append(mounts.input_logs_dir) append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) + container_start_cmd.append(clp_config.container_image_ref) # fmt: off scheduler_cmd = [ @@ -762,7 +762,7 @@ def generic_start_worker( env_vars.extend(aws_env_vars) append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) + container_start_cmd.append(clp_config.container_image_ref) worker_cmd = [ "python3", @@ -969,7 +969,7 @@ def start_webui( necessary_mounts.append(mounts.stream_output_dir) append_docker_options(container_cmd, necessary_mounts, env_vars) - container_cmd.append(clp_config.execution_container) + container_cmd.append(clp_config.container_image_ref) node_cmd = [ str(CONTAINER_CLP_HOME / "bin" / "node-22"), @@ -1023,7 +1023,7 @@ def start_reducer( mounts.generated_config_file, ] append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) + container_start_cmd.append(clp_config.container_image_ref) # fmt: off reducer_cmd = [ @@ -1101,7 +1101,7 @@ def start_garbage_collector( env_vars.extend(aws_env_vars) append_docker_options(container_start_cmd, necessary_mounts, env_vars) - container_start_cmd.append(clp_config.execution_container) + container_start_cmd.append(clp_config.container_image_ref) # fmt: off garbage_collector_cmd = [ diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 1ac4d7083f..ba8a8728cf 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -83,14 +83,13 @@ COMPRESSION_JOBS_TABLE_NAME = "compression_jobs" COMPRESSION_TASKS_TABLE_NAME = "compression_tasks" -OS_RELEASE_FILE_PATH = pathlib.Path("etc") / "os-release" - CLP_DEFAULT_CREDENTIALS_FILE_PATH = pathlib.Path("etc") / "credentials.yml" CLP_DEFAULT_DATA_DIRECTORY_PATH = pathlib.Path("var") / "data" CLP_DEFAULT_DATASET_NAME = "default" CLP_METADATA_TABLE_PREFIX = "clp_" +CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH = pathlib.Path("clp-package-image.id") CLP_SHARED_CONFIG_FILENAME = ".clp-config.yml" - +CLP_VERSION_FILE_PATH = pathlib.Path("VERSION") # Environment variable names CLP_DB_USER_ENV_VAR_NAME = "CLP_DB_USER" @@ -876,7 +875,7 @@ def _get_env_var(name: str) -> str: class CLPConfig(BaseModel): - execution_container: Optional[str] = None + container_image_ref: Optional[str] = None logs_input: Union[FsIngestionConfig, S3IngestionConfig] = FsIngestionConfig() @@ -902,7 +901,10 @@ class CLPConfig(BaseModel): logs_directory: pathlib.Path = pathlib.Path("var") / "log" aws_config_directory: Optional[pathlib.Path] = None - _os_release_file_path: pathlib.Path = PrivateAttr(default=OS_RELEASE_FILE_PATH) + _container_image_id_path: pathlib.Path = PrivateAttr( + default=CLP_PACKAGE_CONTAINER_IMAGE_ID_PATH + ) + _version_file_path: pathlib.Path = PrivateAttr(default=CLP_VERSION_FILE_PATH) @field_validator("aws_config_directory") @classmethod @@ -919,7 +921,10 @@ def make_config_paths_absolute(self, clp_home: pathlib.Path): self.stream_output.storage.make_config_paths_absolute(clp_home) self.data_directory = make_config_path_absolute(clp_home, self.data_directory) self.logs_directory = make_config_path_absolute(clp_home, self.logs_directory) - self._os_release_file_path = make_config_path_absolute(clp_home, self._os_release_file_path) + self._container_image_id_path = make_config_path_absolute( + clp_home, self._container_image_id_path + ) + self._version_file_path = make_config_path_absolute(clp_home, self._version_file_path) def validate_logs_input_config(self): logs_input_type = self.logs_input.type @@ -1007,22 +1012,18 @@ def validate_aws_config_dir(self): "aws_config_directory should not be set when profile authentication is not used" ) - def load_execution_container_name(self): - if self.execution_container is not None: + def load_container_image_ref(self): + if self.container_image_ref is not None: # Accept configured value for debug purposes return - os_release = dotenv_values(self._os_release_file_path) - if "ubuntu" == os_release["ID"]: - self.execution_container = ( - f"clp-execution-x86-{os_release['ID']}-{os_release['VERSION_CODENAME']}:main" - ) + if self._container_image_id_path.exists(): + with open(self._container_image_id_path) as image_id_file: + self.container_image_ref = image_id_file.read().strip() else: - raise NotImplementedError( - f"Unsupported OS {os_release['ID']} in {OS_RELEASE_FILE_PATH}" - ) - - self.execution_container = "ghcr.io/y-scope/clp/" + self.execution_container + with open(self._version_file_path) as version_file: + clp_package_version = version_file.read().strip() + self.container_image_ref = f"ghcr.io/y-scope/clp/clp-package:{clp_package_version}" def get_shared_config_file_path(self) -> pathlib.Path: return self.logs_directory / CLP_SHARED_CONFIG_FILENAME diff --git a/docs/src/dev-docs/tooling-containers.md b/docs/src/dev-docs/tooling-containers.md index aa7b7740cc..a2b4c21853 100644 --- a/docs/src/dev-docs/tooling-containers.md +++ b/docs/src/dev-docs/tooling-containers.md @@ -129,24 +129,6 @@ Jammy x86 environment. components/core/tools/docker-images/clp-core-ubuntu-jammy ``` -## clp-execution-x86-ubuntu-jammy - -An image containing the dependencies necessary to run the CLP package in an Ubuntu Jammy x86 -environment. - -* [GitHub Packages page][exe-ubuntu-jammy] -* Pull command: - - ```bash - docker pull ghcr.io/y-scope/clp/clp-execution-x86-ubuntu-jammy:main - ``` - -* Path: - - ```text - tools/docker-images/clp-execution-base-ubuntu-jammy - ``` - ## clp-package An image containing the CLP package. @@ -162,7 +144,6 @@ An image containing the CLP package. [core-deps-musllinux_1_2-x86_64]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-musllinux_1_2 [core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy -[exe-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-execution-x86-ubuntu-jammy [gh-packages]: https://github.com/orgs/y-scope/packages?repo_name=clp [manylinux_2_28]: https://github.com/pypa/manylinux?tab=readme-ov-file#manylinux_2_28-almalinux-8-based [musllinux_1_2]: https://github.com/pypa/manylinux?tab=readme-ov-file#musllinux_1_2-alpine-linux-322-based-313-compatible diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md index 418a3f5648..4d52694b3e 100644 --- a/docs/src/dev-docs/tooling-gh-workflows.md +++ b/docs/src/dev-docs/tooling-gh-workflows.md @@ -88,11 +88,6 @@ need it for one OS since users can use the container on any OS. This workflow builds CLP-core on macOS and runs its unit tests. -## clp-execution-image-build - -This workflow builds a container image that contains the dependencies necessary to run the CLP -package. - ## clp-lint This workflow runs linting checks on the codebase. diff --git a/taskfile.yaml b/taskfile.yaml index 7bea8bcf83..f1873b9d91 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -52,7 +52,7 @@ vars: tasks: default: - deps: ["package"] + deps: ["docker-images:package"] clean: cmds: @@ -113,7 +113,6 @@ tasks: - "{{.G_CORE_COMPONENT_BUILD_DIR}}/indexer" - "{{.G_CORE_COMPONENT_BUILD_DIR}}/reducer-server" - "{{.TASKFILE}}" - - "/etc/os-release" - "components/clp-package-utils/dist/*.whl" - "components/clp-py-utils/dist/*.whl" - "components/job-orchestration/dist/*.whl" @@ -135,7 +134,6 @@ tasks: cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - "rsync -a components/package-template/src/ '{{.OUTPUT_DIR}}'" - - "rsync --copy-links /etc/os-release '{{.OUTPUT_DIR}}/etc/os-release'" - "mkdir -p '{{.OUTPUT_DIR}}/lib/python3/site-packages'" - |- . "{{.G_PACKAGE_VENV_DIR}}/bin/activate" diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 9a546f713c..821c1a91ae 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -12,3 +12,4 @@ tasks: - ":package" cmds: - "./build.sh" + - "rsync --archive '{{.G_BUILD_DIR}}/clp-package-image.id' '{{.G_PACKAGE_BUILD_DIR}}'" diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile deleted file mode 100644 index 5e7b36dfeb..0000000000 --- a/tools/docker-images/clp-execution-base-ubuntu-jammy/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM ubuntu:jammy AS base - -WORKDIR /root - -RUN mkdir -p ./tools/docker-images/clp-execution-base-ubuntu-jammy -ADD ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts - -RUN ./tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh - -# Remove cached files -RUN apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Flatten the image -FROM scratch -COPY --from=base / / diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh deleted file mode 100755 index d0e099419e..0000000000 --- a/tools/docker-images/clp-execution-base-ubuntu-jammy/build.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -# Exit on any error -set -e - -# Error on undefined variable -set -u - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -repo_root=${script_dir}/../../../ - -arch=$(uname -m) - -if [ "$arch" = "x86_64" ]; then - arch_name="x86" -elif [ "$arch" = "aarch64" ]; then - arch_name="arm64" -else - echo "Error: Unsupported architecture - $arch" - exit 1 -fi - -build_cmd=( - docker build - --tag clp-execution-${arch_name}-ubuntu-jammy:dev - "$repo_root" - --file "${script_dir}/Dockerfile" -) - -if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; -then - build_cmd+=( - --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" - --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" - ) -fi - -"${build_cmd[@]}" diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh deleted file mode 100755 index 9ced27320f..0000000000 --- a/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -# Exit on any error -set -e - -# Error on undefined variable -set -u - -apt-get update -DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - ca-certificates \ - libcurl4 \ - libmariadb3 \ - python3