From cf8c981844ece7d9a988a2f61dfa27800ba45c6a Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Mon, 27 Oct 2025 14:13:35 +0000 Subject: [PATCH 1/7] fix(infra): Fix race condition in parallel base image builds The parallel execution of base image builds was causing a race condition, where dependent images (e.g., `base-builder`) were starting their build process before their base images (e.g., `base-clang`) had finished building in the same pipeline. This resulted in the builder pulling the previously existing, and incorrect, base image from the registry (e.g., an Ubuntu 20.04-based image instead of the new 24.04 version). This commit fixes the issue by introducing an `IMAGE_DEPENDENCIES` map that explicitly defines the build order. The `get_base_image_steps` function now uses this map to add `waitFor` clauses to the Google Cloud Build steps, ensuring that images are built sequentially according to their dependency graph. --- infra/build/functions/base_images.py | 43 ++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/infra/build/functions/base_images.py b/infra/build/functions/base_images.py index 0df75b731a63..ba556611f5f3 100644 --- a/infra/build/functions/base_images.py +++ b/infra/build/functions/base_images.py @@ -47,6 +47,23 @@ # This version will receive the ':v1' tag. DEFAULT_VERSION = 'legacy' +# Defines the dependency graph for base images. +IMAGE_DEPENDENCIES = { + 'base-clang': ['base-image'], + 'base-clang-full': ['base-clang'], + 'base-builder': ['base-clang'], + 'base-builder-go': ['base-builder'], + 'base-builder-javascript': ['base-builder'], + 'base-builder-jvm': ['base-builder'], + 'base-builder-python': ['base-builder'], + 'base-builder-ruby': ['base-builder'], + 'base-builder-rust': ['base-builder'], + 'base-builder-swift': ['base-builder'], + 'base-runner': ['base-image'], + 'base-runner-debug': ['base-runner'], + 'indexer': ['base-clang-full'], +} + class ImageConfig: """Configuration for a specific base image version.""" @@ -85,6 +102,8 @@ def _resolve_dockerfile(self) -> str: if os.path.exists(versioned_dockerfile): logging.info('Using versioned Dockerfile: %s', versioned_dockerfile) return versioned_dockerfile + raise FileNotFoundError( + f'Versioned Dockerfile not found for {self.name}:{self.version}') legacy_dockerfile = os.path.join(self.path, 'Dockerfile') logging.info('Using legacy Dockerfile: %s', legacy_dockerfile) @@ -156,6 +175,8 @@ def full_image_name_with_tag(self) -> str: def get_base_image_steps(images: Sequence[ImageConfig]) -> list[dict]: """Returns build steps for a given list of image configurations.""" steps = [build_lib.get_git_clone_step()] + build_ids = {} + for image_config in images: # The final tag is ':v1' for the default version, or the version name # (e.g., ':ubuntu-24-04') for others. @@ -167,11 +188,23 @@ def get_base_image_steps(images: Sequence[ImageConfig]) -> list[dict]: tags.append(f'{IMAGE_NAME_PREFIX}{image_config.name}:latest') dockerfile_path = os.path.join('oss-fuzz', image_config.dockerfile_path) - steps.append( - build_lib.get_docker_build_step(tags, - image_config.path, - dockerfile_path=dockerfile_path, - build_args=image_config.build_args)) + step = build_lib.get_docker_build_step( + tags, + image_config.path, + dockerfile_path=dockerfile_path, + build_args=image_config.build_args) + + # Check for dependencies and add 'waitFor' if necessary. + dependencies = IMAGE_DEPENDENCIES.get(image_config.name, []) + wait_for = [ + build_ids[dep] for dep in dependencies if dep in build_ids + ] + if wait_for: + step['waitFor'] = wait_for + + build_ids[image_config.name] = step['id'] + steps.append(step) + return steps From 4ad2ac753db39ad4bd0d638e79fd5891e320b3d7 Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Mon, 27 Oct 2025 16:31:09 +0000 Subject: [PATCH 2/7] fix(docker): Update Dockerfile to install libssl-dev before running dependency script --- infra/base-images/base-runner/ubuntu-24-04.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile index e1ac15e2a17f..f564149eedae 100644 --- a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile +++ b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile @@ -54,7 +54,8 @@ RUN ldconfig && \ ln -s /usr/local/bin/python3.11 /usr/local/bin/python COPY install_deps_ubuntu_24_04.sh / -RUN /install_deps_ubuntu_24_04.sh && rm /install_deps_ubuntu_24_04.sh +RUN apt-get update && apt-get install -y libssl-dev && \ + /install_deps_ubuntu_24_04.sh && rm /install_deps_ubuntu_24_04.sh ENV CODE_COVERAGE_SRC=/opt/code_coverage # Pin coverage to the same as in the base builder: From 5ba63ea289ed7bf8c9adc40f3bd2c5c558e69f8c Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Mon, 27 Oct 2025 16:45:33 +0000 Subject: [PATCH 3/7] refactor(base_images): Simplify argument formatting in get_base_image_steps function --- infra/build/functions/base_images.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/infra/build/functions/base_images.py b/infra/build/functions/base_images.py index ba556611f5f3..9d39f72aa6df 100644 --- a/infra/build/functions/base_images.py +++ b/infra/build/functions/base_images.py @@ -188,17 +188,14 @@ def get_base_image_steps(images: Sequence[ImageConfig]) -> list[dict]: tags.append(f'{IMAGE_NAME_PREFIX}{image_config.name}:latest') dockerfile_path = os.path.join('oss-fuzz', image_config.dockerfile_path) - step = build_lib.get_docker_build_step( - tags, - image_config.path, - dockerfile_path=dockerfile_path, - build_args=image_config.build_args) + step = build_lib.get_docker_build_step(tags, + image_config.path, + dockerfile_path=dockerfile_path, + build_args=image_config.build_args) # Check for dependencies and add 'waitFor' if necessary. dependencies = IMAGE_DEPENDENCIES.get(image_config.name, []) - wait_for = [ - build_ids[dep] for dep in dependencies if dep in build_ids - ] + wait_for = [build_ids[dep] for dep in dependencies if dep in build_ids] if wait_for: step['waitFor'] = wait_for From 68ef1ee6e35e9611c8c81fa67f01042d02e74f3a Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Mon, 27 Oct 2025 19:13:06 +0000 Subject: [PATCH 4/7] refactor(base_images): Improve formatting of get_base_image_steps function --- infra/build/functions/base_images.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/infra/build/functions/base_images.py b/infra/build/functions/base_images.py index 9d39f72aa6df..42c1adc32506 100644 --- a/infra/build/functions/base_images.py +++ b/infra/build/functions/base_images.py @@ -188,10 +188,11 @@ def get_base_image_steps(images: Sequence[ImageConfig]) -> list[dict]: tags.append(f'{IMAGE_NAME_PREFIX}{image_config.name}:latest') dockerfile_path = os.path.join('oss-fuzz', image_config.dockerfile_path) - step = build_lib.get_docker_build_step(tags, - image_config.path, - dockerfile_path=dockerfile_path, - build_args=image_config.build_args) + step = build_lib.get_docker_build_step( + tags, + image_config.path, + dockerfile_path=dockerfile_path, + build_args=image_config.build_args) # Check for dependencies and add 'waitFor' if necessary. dependencies = IMAGE_DEPENDENCIES.get(image_config.name, []) From 27a815efa7bf7b3842f618a0af6def3397984a16 Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Mon, 27 Oct 2025 19:15:20 +0000 Subject: [PATCH 5/7] refactor(base_images): Improve formatting of get_base_image_steps function --- infra/build/functions/base_images.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/infra/build/functions/base_images.py b/infra/build/functions/base_images.py index 42c1adc32506..9d39f72aa6df 100644 --- a/infra/build/functions/base_images.py +++ b/infra/build/functions/base_images.py @@ -188,11 +188,10 @@ def get_base_image_steps(images: Sequence[ImageConfig]) -> list[dict]: tags.append(f'{IMAGE_NAME_PREFIX}{image_config.name}:latest') dockerfile_path = os.path.join('oss-fuzz', image_config.dockerfile_path) - step = build_lib.get_docker_build_step( - tags, - image_config.path, - dockerfile_path=dockerfile_path, - build_args=image_config.build_args) + step = build_lib.get_docker_build_step(tags, + image_config.path, + dockerfile_path=dockerfile_path, + build_args=image_config.build_args) # Check for dependencies and add 'waitFor' if necessary. dependencies = IMAGE_DEPENDENCIES.get(image_config.name, []) From b579b368b602e7677e0993d9c110e47885a66f07 Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Mon, 27 Oct 2025 19:25:37 +0000 Subject: [PATCH 6/7] revert(base-images): Remove unnecessary libssl-dev installation This reverts the change that added libssl-dev to the ubuntu-24-04 base-runner image. This dependency was found to be unnecessary as the build works without it, as seen in trial builds. --- infra/base-images/base-runner/ubuntu-24-04.Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile index f564149eedae..e1ac15e2a17f 100644 --- a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile +++ b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile @@ -54,8 +54,7 @@ RUN ldconfig && \ ln -s /usr/local/bin/python3.11 /usr/local/bin/python COPY install_deps_ubuntu_24_04.sh / -RUN apt-get update && apt-get install -y libssl-dev && \ - /install_deps_ubuntu_24_04.sh && rm /install_deps_ubuntu_24_04.sh +RUN /install_deps_ubuntu_24_04.sh && rm /install_deps_ubuntu_24_04.sh ENV CODE_COVERAGE_SRC=/opt/code_coverage # Pin coverage to the same as in the base builder: From fb1ff2c6b777f119768da7bb44eb15b0b5e0e6e9 Mon Sep 17 00:00:00 2001 From: Matheus Hunsche Date: Tue, 28 Oct 2025 14:37:09 +0000 Subject: [PATCH 7/7] fix(base_images): Update base-runner dependencies to include base-builder --- infra/build/functions/base_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/build/functions/base_images.py b/infra/build/functions/base_images.py index 9d39f72aa6df..b1a90ff09132 100644 --- a/infra/build/functions/base_images.py +++ b/infra/build/functions/base_images.py @@ -59,7 +59,7 @@ 'base-builder-ruby': ['base-builder'], 'base-builder-rust': ['base-builder'], 'base-builder-swift': ['base-builder'], - 'base-runner': ['base-image'], + 'base-runner': ['base-image', 'base-builder'], 'base-runner-debug': ['base-runner'], 'indexer': ['base-clang-full'], }