From 4fd6b73b310706765fc9122d09710c4e42d7f53c Mon Sep 17 00:00:00 2001 From: Oliver Chang Date: Fri, 17 Oct 2025 04:29:10 +1100 Subject: [PATCH] Revert "feat(infra): Add base_os_version to support parallel builds (#14128)" This reverts commit eb47f56bd15626ab4ae8727bc435148dd2f9857c. --- .github/workflows/check_base_os.yml | 61 ------------ infra/build/functions/build_lib.py | 18 +--- infra/build/functions/build_project.py | 5 +- infra/build/functions/trial_build.py | 3 +- infra/build/functions/trial_build_test.py | 2 +- infra/ci/check_base_os.py | 111 ---------------------- infra/helper.py | 90 ++++-------------- 7 files changed, 27 insertions(+), 263 deletions(-) delete mode 100644 .github/workflows/check_base_os.yml delete mode 100644 infra/ci/check_base_os.py diff --git a/.github/workflows/check_base_os.yml b/.github/workflows/check_base_os.yml deleted file mode 100644 index 26dd544ac286..000000000000 --- a/.github/workflows/check_base_os.yml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -name: 'Check Base OS Consistency' - -on: - pull_request: - paths: - - 'projects/**' - -jobs: - check-consistency: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch all history to compare with main - - - name: Get changed project directories - id: changed-projects - run: | - # Get the list of changed files compared to the target branch - # and filter for unique directories under 'projects/'. - CHANGED_DIRS=$(git diff --name-only ${{ github.base_ref }} ${{ github.head_ref }} | \ - grep '^projects/' | \ - xargs -n 1 dirname | \ - sort -u) - echo "changed_dirs=${CHANGED_DIRS}" >> $GITHUB_OUTPUT - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install dependencies - run: pip install PyYAML - - - name: Check each modified project - if: steps.changed-projects.outputs.changed_dirs != '' - run: | - EXIT_CODE=0 - for project_dir in ${{ steps.changed-projects.outputs.changed_dirs }}; - do - echo "--- Checking ${project_dir} ---" - python3 infra/ci/check_base_os.py "${project_dir}" || EXIT_CODE=$? - done - exit $EXIT_CODE diff --git a/infra/build/functions/build_lib.py b/infra/build/functions/build_lib.py index f00ab7c3c8b4..54d1aa8ef1b3 100644 --- a/infra/build/functions/build_lib.py +++ b/infra/build/functions/build_lib.py @@ -685,24 +685,12 @@ def get_gcb_url(build_id, cloud_project='oss-fuzz'): f'{build_id}?project={cloud_project}') -def get_runner_image_name(test_image_suffix, base_image_tag=None): - """Returns the runner image that should be used. - - Returns the testing image if |test_image_suffix|. - """ +def get_runner_image_name(test_image_suffix): + """Returns the runner image that should be used. Returns the testing image if + |test_image_suffix|.""" image = f'gcr.io/{BASE_IMAGES_PROJECT}/base-runner' - - # For trial builds, the version is embedded in the suffix. if test_image_suffix: image += '-' + test_image_suffix - return image - - # For local/manual runs, the version is passed as a tag. - # Only add a tag if it's specified and not 'legacy', as 'legacy' implies - # 'latest', which is the default behavior. - if base_image_tag and base_image_tag != 'legacy': - image += ':' + base_image_tag - return image diff --git a/infra/build/functions/build_project.py b/infra/build/functions/build_project.py index cfc14e6ac884..e95b5065189e 100755 --- a/infra/build/functions/build_project.py +++ b/infra/build/functions/build_project.py @@ -73,7 +73,6 @@ class Config: testing: bool = False test_image_suffix: Optional[str] = None - base_image_tag: Optional[str] = None repo: Optional[str] = DEFAULT_OSS_FUZZ_REPO branch: Optional[str] = None parallel: bool = False @@ -454,10 +453,8 @@ def get_build_steps_for_project(project, f'--architecture {build.architecture} {project.name}\\n' + '*' * 80) # Test fuzz targets. - runner_image_name = build_lib.get_runner_image_name( - config.test_image_suffix, config.base_image_tag) test_step = { - 'name': runner_image_name, + 'name': build_lib.get_runner_image_name(config.test_image_suffix), 'env': env, 'args': [ 'bash', '-c', diff --git a/infra/build/functions/trial_build.py b/infra/build/functions/trial_build.py index 849effed8515..a11280882fa9 100644 --- a/infra/build/functions/trial_build.py +++ b/infra/build/functions/trial_build.py @@ -321,7 +321,6 @@ def _do_test_builds(args, test_image_suffix, end_time, version_tag): config = build_project.Config(testing=True, test_image_suffix=test_image_suffix, - base_image_tag=version_tag, repo=args.repo, branch=args.branch, parallel=False, @@ -433,7 +432,7 @@ def _do_build_type_builds(args, config, credentials, build_type, projects): credentials, build_type.type_name, extra_tags=tags, - timeout=PROJECT_BUILD_TIMEOUT))['id'] + timeout=PROJECT_BUILD_TIMEOUT)) time.sleep(1) # Avoid going over 75 requests per second limit. except Exception as error: # pylint: disable=broad-except # Handle flake. diff --git a/infra/build/functions/trial_build_test.py b/infra/build/functions/trial_build_test.py index 9b74791c6702..9cd20207b1b9 100644 --- a/infra/build/functions/trial_build_test.py +++ b/infra/build/functions/trial_build_test.py @@ -73,7 +73,7 @@ def test_build_steps_correct(self, mock_gcb_build_and_push_images, del mock_wait_on_builds self.maxDiff = None # pylint: disable=invalid-name build_id = 1 - mock_run_build.return_value = {'id': build_id} + mock_run_build.return_value = build_id branch_name = 'mybranch' project = 'skcms' args = [ diff --git a/infra/ci/check_base_os.py b/infra/ci/check_base_os.py deleted file mode 100644 index 90f2ccfb2e28..000000000000 --- a/infra/ci/check_base_os.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -""" -A CI script to ensure that the base OS version specified in a project's -project.yaml file matches the FROM line in its Dockerfile. -""" - -import os -import sys -import yaml - -# Defines the base OS versions that are currently supported for use in project.yaml. -# For now, only 'legacy' is permitted. This list will be expanded as new -# base images are rolled out. -SUPPORTED_VERSIONS = [ - 'legacy', - # 'ubuntu-20-04', - # 'ubuntu-24-04', -] - -# A map from the base_os_version in project.yaml to the expected Dockerfile -# FROM tag. -BASE_OS_TO_DOCKER_TAG = { - 'legacy': 'latest', - 'ubuntu-20-04': 'ubuntu-20-04', - 'ubuntu-24-04': 'ubuntu-24-04', -} - - -def main(): - """Checks the Dockerfile FROM tag against the project's base_os_version.""" - if len(sys.argv) < 2: - print(f'Usage: {sys.argv[0]} ', file=sys.stderr) - return 1 - - project_path = sys.argv[1] - project_yaml_path = os.path.join(project_path, 'project.yaml') - dockerfile_path = os.path.join(project_path, 'Dockerfile') - - # 1. Get the base_os_version from project.yaml, defaulting to 'legacy'. - base_os_version = 'legacy' - if os.path.exists(project_yaml_path): - with open(project_yaml_path) as f: - config = yaml.safe_load(f) - if config and 'base_os_version' in config: - base_os_version = config['base_os_version'] - - # 2. Validate that the version is currently supported. - if base_os_version not in SUPPORTED_VERSIONS: - print( - f'Error: base_os_version "{base_os_version}" is not yet supported. ' - f'The currently supported versions are: "{", ".join(SUPPORTED_VERSIONS)}"', - file=sys.stderr) - return 1 - - # 3. Get the expected Dockerfile tag from our mapping. - expected_tag = BASE_OS_TO_DOCKER_TAG[base_os_version] - - # 4. Read the Dockerfile and find the tag in the FROM line. - if not os.path.exists(dockerfile_path): - print(f'Error: Dockerfile not found at {dockerfile_path}', file=sys.stderr) - return 1 - - dockerfile_tag = '' - with open(dockerfile_path) as f: - for line in f: - if line.strip().startswith('FROM'): - try: - if ':' not in line: - print( - f'Error: Malformed FROM line in Dockerfile (missing tag): {line.strip()}', - file=sys.stderr) - return 1 - dockerfile_tag = line.split(':')[1].strip() - except IndexError: - print(f'Error: Could not parse tag from Dockerfile FROM line: {line}', - file=sys.stderr) - return 1 - break - - # 5. Compare and report. - if dockerfile_tag != expected_tag: - print( - f'Error: Mismatch found in {project_path}.\n' - f' - project.yaml (base_os_version): "{base_os_version}" (expects Dockerfile tag "{expected_tag}")\n' - f' - Dockerfile FROM tag: "{dockerfile_tag}"\n' - f'Please align the Dockerfile\'s FROM line to use the tag "{expected_tag}".', - file=sys.stderr) - return 1 - - print( - f'Success: {project_path} is consistent (base_os_version: "{base_os_version}", Dockerfile tag: "{dockerfile_tag}").' - ) - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/infra/helper.py b/infra/helper.py index 5b32e846bded..17e0b1c92796 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -31,7 +31,6 @@ import sys import tempfile import urllib.request -import yaml import constants import templates @@ -41,23 +40,6 @@ BASE_RUNNER_IMAGE = 'gcr.io/oss-fuzz-base/base-runner' - -def _get_base_runner_image(args, debug=False): - """Returns the base runner image to use.""" - image = BASE_RUNNER_IMAGE - if debug: - image += '-debug' - - tag = 'latest' - if hasattr(args, 'base_image_tag') and args.base_image_tag: - tag = args.base_image_tag - elif hasattr(args, 'project') and args.project: - if args.project.base_os_version != 'legacy': - tag = args.project.base_os_version - - return f'{image}:{tag}' - - BASE_IMAGES = { 'generic': [ 'gcr.io/oss-fuzz-base/base-image', @@ -161,28 +143,15 @@ def language(self): return constants.DEFAULT_LANGUAGE with open(project_yaml_path) as file_handle: - config = yaml.safe_load(file_handle) - if config and 'language' in config: - return config['language'] + content = file_handle.read() + for line in content.splitlines(): + match = PROJECT_LANGUAGE_REGEX.match(line) + if match: + return match.group(1) logger.warning('Language not specified in project.yaml. Assuming c++.') return constants.DEFAULT_LANGUAGE - @property - def base_os_version(self): - """Returns the project's base OS version.""" - project_yaml_path = os.path.join(self.build_integration_path, - 'project.yaml') - if not os.path.exists(project_yaml_path): - return 'legacy' - - with open(project_yaml_path) as file_handle: - config = yaml.safe_load(file_handle) - version = 'legacy' - if config and 'base_os_version' in config: - version = config['base_os_version'] - return version - @property def coverage_extra_args(self): """Returns project coverage extra args.""" @@ -395,7 +364,6 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_engine_args(check_build_parser, choices=constants.ENGINES) _add_sanitizer_args(check_build_parser, choices=constants.SANITIZERS) _add_environment_args(check_build_parser) - _add_base_image_tag_args(check_build_parser) check_build_parser.add_argument('project', help='name of the project or path (external)') check_build_parser.add_argument('fuzzer_name', @@ -431,7 +399,6 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_engine_args(run_fuzzer_parser) _add_sanitizer_args(run_fuzzer_parser) _add_environment_args(run_fuzzer_parser) - _add_base_image_tag_args(run_fuzzer_parser) _add_external_project_args(run_fuzzer_parser) run_fuzzer_parser.add_argument( '--corpus-dir', help='directory to store corpus for the fuzz target') @@ -498,7 +465,6 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals nargs='*') _add_external_project_args(coverage_parser) _add_architecture_args(coverage_parser) - _add_base_image_tag_args(coverage_parser) introspector_parser = subparsers.add_parser( 'introspector', @@ -558,7 +524,6 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_environment_args(reproduce_parser) _add_external_project_args(reproduce_parser) _add_architecture_args(reproduce_parser) - _add_base_image_tag_args(reproduce_parser) shell_parser = subparsers.add_parser( 'shell', help='Run /bin/bash within the builder container.') @@ -572,7 +537,6 @@ def get_parser(): # pylint: disable=too-many-statements,too-many-locals _add_sanitizer_args(shell_parser) _add_environment_args(shell_parser) _add_external_project_args(shell_parser) - _add_base_image_tag_args(shell_parser) run_clusterfuzzlite_parser = subparsers.add_parser( 'run_clusterfuzzlite', help='Run ClusterFuzzLite on a project.') @@ -618,12 +582,12 @@ def check_project_exists(project): return False -def _check_fuzzer_exists(project, fuzzer_name, args, architecture='x86_64'): +def _check_fuzzer_exists(project, fuzzer_name, architecture='x86_64'): """Checks if a fuzzer exists.""" platform = 'linux/arm64' if architecture == 'aarch64' else 'linux/amd64' command = ['docker', 'run', '--rm', '--platform', platform] command.extend(['-v', '%s:/out' % project.out]) - command.append(_get_base_runner_image(args)) + command.append(BASE_RUNNER_IMAGE) command.extend(['/bin/bash', '-c', 'test -f /out/%s' % fuzzer_name]) @@ -704,12 +668,6 @@ def _add_environment_args(parser): help="set environment variable e.g. VAR=value") -def _add_base_image_tag_args(parser): - """Adds base image tag arg.""" - parser.add_argument('--base-image-tag', - help='The tag of the base-runner image to use.') - - def build_image_impl(project, cache=True, pull=False, architecture='x86_64'): """Builds image.""" image_name = project.name @@ -1106,13 +1064,11 @@ def _add_oss_fuzz_ci_if_needed(env): def check_build(args): """Checks that fuzzers in the container execute without errors.""" - # Access the property to trigger validation early. - _ = args.project.base_os_version if not check_project_exists(args.project): return False if (args.fuzzer_name and not _check_fuzzer_exists( - args.project, args.fuzzer_name, args, args.architecture)): + args.project, args.fuzzer_name, args.architecture)): return False env = [ @@ -1127,8 +1083,7 @@ def check_build(args): env += args.e run_args = _env_to_docker_args(env) + [ - '-v', f'{args.project.out}:/out', '-t', - _get_base_runner_image(args) + '-v', f'{args.project.out}:/out', '-t', BASE_RUNNER_IMAGE ] if args.fuzzer_name: @@ -1364,7 +1319,7 @@ def coverage(args): # pylint: disable=too-many-branches '-v', '%s:/out' % args.project.out, '-t', - _get_base_runner_image(args), + BASE_RUNNER_IMAGE, ]) run_args.append('coverage') @@ -1492,7 +1447,7 @@ def run_fuzzer(args): if not check_project_exists(args.project): return False - if not _check_fuzzer_exists(args.project, args.fuzzer_name, args, + if not _check_fuzzer_exists(args.project, args.fuzzer_name, args.architecture): return False @@ -1523,7 +1478,7 @@ def run_fuzzer(args): '-v', '%s:/out' % args.project.out, '-t', - _get_base_runner_image(args), + BASE_RUNNER_IMAGE, 'run_fuzzer', args.fuzzer_name, ] + args.fuzzer_args) @@ -1614,8 +1569,7 @@ def fuzzbench_measure(args): def reproduce(args): """Reproduces a specific test case from a specific project.""" return reproduce_impl(args.project, args.fuzzer_name, args.valgrind, args.e, - args.fuzzer_args, args.testcase_path, args, - args.architecture) + args.fuzzer_args, args.testcase_path, args.architecture) def reproduce_impl( # pylint: disable=too-many-arguments @@ -1625,30 +1579,29 @@ def reproduce_impl( # pylint: disable=too-many-arguments env_to_add, fuzzer_args, testcase_path, - args, architecture='x86_64', run_function=docker_run, err_result=False): - """Reproduces a specific test case.""" + """Reproduces a testcase in the container.""" if not check_project_exists(project): return err_result - if not _check_fuzzer_exists(project, fuzzer_name, args, architecture): + if not _check_fuzzer_exists(project, fuzzer_name, architecture): return err_result debugger = '' env = ['HELPER=True', 'ARCHITECTURE=' + architecture] - use_debug_image = bool(valgrind) - image_name = _get_base_runner_image(args, debug=use_debug_image) + image_name = 'base-runner' if valgrind: debugger = 'valgrind --tool=memcheck --track-origins=yes --leak-check=full' if debugger: + image_name = 'base-runner-debug' env += ['DEBUGGER=' + debugger] if env_to_add: - env.extend(env_to_add) + env += env_to_add run_args = _env_to_docker_args(env) + [ '-v', @@ -1656,12 +1609,13 @@ def reproduce_impl( # pylint: disable=too-many-arguments '-v', '%s:/testcase' % _get_absolute_path(testcase_path), '-t', - image_name, + 'gcr.io/oss-fuzz-base/%s' % image_name, 'reproduce', fuzzer_name, '-runs=100', ] + fuzzer_args - return run_function(run_args, err_result) + + return run_function(run_args, architecture=architecture) def _validate_project_name(project_name): @@ -1825,8 +1779,6 @@ def index(args): def shell(args): """Runs a shell within a docker image.""" - # Access the property to trigger validation early. - _ = args.project.base_os_version if not build_image_impl(args.project): return False