build_project.py

# Copyright 2020 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
################################################################################
#!/usr/bin/env python3
"""Starts project build on Google Cloud Builder.

Usage: build_project.py <project_dir>
"""

import argparse
from dataclasses import dataclass
import datetime
import json
import logging
import os
import posixpath
import re
import shlex
import sys
from typing import Optional

import oauth2client.client
import six
import yaml

import build_lib

FUZZING_BUILD_TYPE = 'fuzzing'
INDEXER_BUILD_TYPE = 'indexer'

GCB_LOGS_BUCKET = 'oss-fuzz-gcb-logs'
GCB_EXPERIMENT_LOGS_BUCKET = 'oss-fuzz-gcb-experiment-logs'

DEFAULT_ARCHITECTURES = ['x86_64']
DEFAULT_ENGINES = ['libfuzzer', 'afl', 'honggfuzz', 'centipede']
DEFAULT_SANITIZERS = ['address', 'undefined']

LATEST_VERSION_FILENAME = 'latest.version'
LATEST_VERSION_CONTENT_TYPE = 'text/plain'

QUEUE_TTL_SECONDS = 60 * 60 * 24  # 24 hours.

PROJECTS_DIR = os.path.abspath(
    os.path.join(__file__, os.path.pardir, os.path.pardir, os.path.pardir,
                 os.path.pardir, 'projects'))

DEFAULT_OSS_FUZZ_REPO = 'https://github.com/google/oss-fuzz.git'

# Used if build logs are uploaded to a separate place.
LOCAL_BUILD_LOG_PATH = '/workspace/build.log'
BUILD_SUCCESS_MARKER = '/workspace/build.succeeded'

_CACHED_IMAGE = ('us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/'
                 '{name}-ofg-cached-{sanitizer}')
_CACHED_SANITIZERS = ('address', 'coverage')
_INDEXED_CONTAINER_NAME = 'indexed-container'
_TRACING_CONTAINER_NAME = 'tracing-container'


@dataclass
class Config:
  testing: bool = False
  test_image_suffix: Optional[str] = None
  base_image_tag: Optional[str] = None
  repo: Optional[str] = DEFAULT_OSS_FUZZ_REPO
  branch: Optional[str] = None
  parallel: bool = False
  upload: bool = True
  experiment: bool = False
  # TODO(ochang): This should be different per engine+sanitizer combination.
  upload_build_logs: Optional[str] = None
  build_type: Optional[str] = None
  fuzzing_engine: Optional[str] = None
  fuzz_target: Optional[str] = None


# Allow the WORKDIR to be commented out for OSS-Fuzz-Gen, which creates new
# Dockerfiles that inherit from cached verisons of the project images.
# e.g.
#   FROM us-central1-docker.pkg.dev/oss-fuzz/oss-fuzz-gen/proj-ofg-cached-address
#   # WORKDIR foo
#   COPY new_target.c /src/proj/
#
# Because the WORKDIR is already set in the parent image and can be a relative
# path, we can't set it again in the new Dockerfile.
# However, we still need to know what the value is (for GCB), so we leave it
# commented.
WORKDIR_REGEX = re.compile(r'\s*#?\s*WORKDIR\s*([^\s]+)')


class Build:  # pylint: disable=too-few-public-methods
  """Class representing the configuration for a build."""

  def __init__(self, fuzzing_engine, sanitizer, architecture):
    self.fuzzing_engine = fuzzing_engine
    self.sanitizer = sanitizer
    self.architecture = architecture
    self.targets_list_filename = build_lib.get_targets_list_filename(
        self.sanitizer)

  @property
  def is_arm(self):
    """Returns True if CPU architecture is ARM-based."""
    return self.architecture == 'aarch64'

  @property
  def out(self):
    """Returns the out directory for the build."""
    return posixpath.join(
        '/workspace/out/',
        f'{self.fuzzing_engine}-{self.sanitizer}-{self.architecture}')


def get_project_data(project_name):
  """(Local only) Returns a tuple containing the contents of the project.yaml
  and Dockerfile of |project_name|. Raises a FileNotFoundError if there is no
  Dockerfile for |project_name|."""
  project_dir = os.path.join(PROJECTS_DIR, project_name)
  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
  try:
    with open(dockerfile_path) as dockerfile:
      dockerfile = dockerfile.read()
  except FileNotFoundError:
    raise
  project_yaml_path = os.path.join(project_dir, 'project.yaml')
  with open(project_yaml_path, 'r') as project_yaml_file_handle:
    project_yaml_contents = project_yaml_file_handle.read()
  project_yaml = yaml.safe_load(project_yaml_contents)
  return project_yaml, dockerfile


def get_sanitizer_strings(sanitizers):
  """Accepts the sanitizers field from project.yaml where some sanitizers can be
  defined as experimental. Returns a list of sanitizers."""
  processed_sanitizers = []
  for sanitizer in sanitizers:
    if isinstance(sanitizer, six.string_types):
      processed_sanitizers.append(sanitizer)
    elif isinstance(sanitizer, dict):
      processed_sanitizers.extend(sanitizer.keys())

  return processed_sanitizers


def set_default_sanitizer_for_centipede(project_yaml):
  """Adds none as a sanitizer for centipede in yaml if it does not exist yet."""
  # Centipede requires a separate unsanitized binary to use sanitized ones.
  if ('centipede' in project_yaml['fuzzing_engines'] and
      project_yaml['sanitizers'] and 'none' not in project_yaml['sanitizers']):
    project_yaml['sanitizers'].append('none')


class Project:  # pylint: disable=too-many-instance-attributes
  """Class representing an OSS-Fuzz project."""

  def __init__(self, name, project_yaml, dockerfile):
    project_yaml = project_yaml.copy()
    set_yaml_defaults(project_yaml)

    self.name = name
    self.workdir = workdir_from_dockerfile(dockerfile)
    self._sanitizers = project_yaml['sanitizers']
    self.disabled = project_yaml['disabled']
    self.architectures = project_yaml['architectures']
    self.fuzzing_engines = project_yaml['fuzzing_engines']
    self.coverage_extra_args = project_yaml['coverage_extra_args']
    self.labels = project_yaml['labels']
    self.fuzzing_language = project_yaml['language']
    self.run_tests = project_yaml['run_tests']
    self.base_os_version = project_yaml.get('base_os_version', 'legacy')
    if 'main_repo' in project_yaml:
      self.main_repo = project_yaml['main_repo']
    else:
      self.main_repo = ''

    self.indexer_targets = project_yaml.get('indexer', {}).get('targets', [])

    # This is set to enable build infra to use cached images (which are
    # specific to a sanitizer).
    # TODO: find a better way to handle this.
    self.cached_sanitizer = None

    # This is used by OSS-Fuzz-Gen, which generates fake project names for each
    # benchmark. We still need access to the real project name in some cases.
    self.real_name = self.name

  @property
  def sanitizers(self):
    """Returns processed sanitizers."""
    assert isinstance(self._sanitizers, list)
    return get_sanitizer_strings(self._sanitizers)

  @property
  def image(self):
    """Returns the docker image for the project."""
    return f'gcr.io/{build_lib.IMAGE_PROJECT}/{self.name}'

  @property
  def cached_image(self):
    return _CACHED_IMAGE.format(name=self.real_name,
                                sanitizer=self.cached_sanitizer)


def get_last_step_id(steps):
  """Returns the id of the last step in |steps|."""
  return steps[-1]['id']


def set_yaml_defaults(project_yaml):
  """Sets project.yaml's default parameters."""
  project_yaml.setdefault('disabled', False)
  project_yaml.setdefault('architectures', DEFAULT_ARCHITECTURES)
  project_yaml.setdefault('sanitizers', DEFAULT_SANITIZERS)
  project_yaml.setdefault('fuzzing_engines', DEFAULT_ENGINES)
  project_yaml.setdefault('run_tests', True)
  project_yaml.setdefault('coverage_extra_args', '')
  project_yaml.setdefault('labels', {})
  # Adds 'none' as a sanitizer for centipede to the project yaml by default,
  # because Centipede always requires a separate build of unsanitized binary.
  set_default_sanitizer_for_centipede(project_yaml)


def is_supported_configuration(build):
  """Check if the given configuration is supported."""
  fuzzing_engine_info = build_lib.ENGINE_INFO[build.fuzzing_engine]
  if build.architecture == 'i386' and build.sanitizer != 'address':
    return False
  # TODO(jonathanmetzman): UBSan should be easy to support.
  if build.architecture == 'aarch64' and (build.sanitizer
                                          not in {'address', 'hwaddress'}):
    return False
  return (build.sanitizer in fuzzing_engine_info.supported_sanitizers and
          build.architecture in fuzzing_engine_info.supported_architectures)


def workdir_from_dockerfile(dockerfile):
  """Parses WORKDIR from the Dockerfile."""
  dockerfile_lines = dockerfile.split('\n')
  for line in dockerfile_lines:
    match = re.match(WORKDIR_REGEX, line)
    if match:
      # We need to escape '$' since they're used for subsitutions in Container
      # Builer builds.
      return match.group(1).replace('$', '$$')

  return '/src'


def get_datetime_now():
  """Returns datetime.datetime.now(). Used for mocking."""
  return datetime.datetime.now()


def get_env(fuzzing_language, build, project_name=None):
  """Returns an environment for building. The environment is returned as a list
  and is suitable for use as the "env" parameter in a GCB build step. The
  environment variables are based on the values of |fuzzing_language| and
  |build."""
  env_dict = {
      'FUZZING_LANGUAGE': fuzzing_language,
      'FUZZING_ENGINE': build.fuzzing_engine,
      'SANITIZER': build.sanitizer,
      'ARCHITECTURE': build.architecture,
      # Set HOME so that it doesn't point to a persisted volume (see
      # https://github.com/google/oss-fuzz/issues/6035).
      'HOME': '/root',
      'OUT': build.out,
  }
  if project_name is not None:
    env_dict['PROJECT_NAME'] = project_name
  return list(sorted([f'{key}={value}' for key, value in env_dict.items()]))


def get_compile_step(project,
                     build,
                     env,
                     parallel,
                     upload_build_logs=None,
                     allow_failure=False):
  """Returns the GCB step for compiling |projects| fuzzers using |env|. The type
  of build is specified by |build|."""
  failure_msg = (
      '*' * 80 + '\\nFailed to build.\\nTo reproduce, run:\\n'
      f'python infra/helper.py build_image {project.name}\\n'
      'python infra/helper.py build_fuzzers --sanitizer '
      f'{build.sanitizer} --engine {build.fuzzing_engine} --architecture '
      f'{build.architecture} {project.name}\\n' + '*' * 80)
  compile_output_redirect = ''

  if upload_build_logs:
    # Also write a build success marker because this step needs to succeed first
    # for a subsequent step to upload the log.
    compile_output_redirect = (
        f'&> {LOCAL_BUILD_LOG_PATH} && touch {BUILD_SUCCESS_MARKER}')

  compile_step = {
      'name': project.image,
      'env': env,
      'args': [
          'bash',
          '-c',
          # Remove /out to make sure there are non instrumented binaries.
          # `cd /src && cd {workdir}` (where {workdir} is parsed from the
          # Dockerfile). Container Builder overrides our workdir so we need
          # to add this step to set it back.
          (f'rm -r /out && cd /src && cd {project.workdir} && '
           f'mkdir -p {build.out} && compile {compile_output_redirect}|| '
           f'(echo "{failure_msg}" && false)'),
      ],
      'id': get_id('compile', build),
  }

  if upload_build_logs or allow_failure:
    # The failure will be reported in a subsequent step.
    compile_step['allowFailure'] = True

  build_lib.dockerify_run_step(compile_step,
                               build,
                               use_architecture_image_name=build.is_arm)
  maybe_add_parallel(compile_step, build_lib.get_srcmap_step_id(), parallel)
  return compile_step


def maybe_add_parallel(step, wait_for_id, parallel):
  """Makes |step| run immediately after |wait_for_id| if |parallel|. Mutates
  |step|."""
  if not parallel:
    return
  step['waitFor'] = wait_for_id


def get_id(step_type, build):
  """Returns a unique step id based on |step_type| and |build|. Useful for
  parallelizing builds."""
  return (f'{step_type}-{build.fuzzing_engine}-{build.sanitizer}'
          f'-{build.architecture}')


def get_build_steps(  # pylint: disable=too-many-locals, too-many-statements, too-many-branches, too-many-arguments
    project_name,
    project_yaml,
    dockerfile,
    config,
    additional_env=None,
    use_caching=False,
    timestamp=None):
  """Returns build steps for project."""

  project = Project(project_name, project_yaml, dockerfile)
  return get_build_steps_for_project(project,
                                     config,
                                     additional_env=additional_env,
                                     use_caching=use_caching,
                                     timestamp=timestamp), None


def get_build_steps_for_project(project,
                                config,
                                additional_env=None,
                                use_caching=False,
                                timestamp=None):
  """Returns build steps for project."""

  if not timestamp:
    timestamp = get_datetime_now()

  timestamp = timestamp.strftime('%Y%m%d%H%M')

  if use_caching:
    # For cached builds: the cache images are sanitizer-specific, so we need to
    # do a rebuild prior to each compile.
    build_steps = []
  else:
    # Non-cached builds just use a single builder image to build all sanitizers.
    build_steps = build_lib.get_project_image_steps(
        project.name,
        project.image,
        project.fuzzing_language,
        config=config,
        architectures=project.architectures,
        experiment=config.experiment)

  # Sort engines to make AFL first to test if libFuzzer has an advantage in
  # finding bugs first since it is generally built first.
  for fuzzing_engine in sorted(project.fuzzing_engines):
    # Sort sanitizers and architectures so order is determinisitic (good for
    # tests).
    for sanitizer in sorted(project.sanitizers):
      if use_caching and sanitizer in _CACHED_SANITIZERS:
        project.cached_sanitizer = sanitizer
        build_steps.extend(
            build_lib.get_project_image_steps(
                project.name,
                project.image,
                project.fuzzing_language,
                config=config,
                architectures=project.architectures,
                experiment=config.experiment,
                cache_image=project.cached_image))

      # Build x86_64 before i386.
      for architecture in reversed(sorted(project.architectures)):
        build = Build(fuzzing_engine, sanitizer, architecture)
        if not is_supported_configuration(build):
          continue

        env = get_env(project.fuzzing_language, build)
        if additional_env:
          env.extend(additional_env)

        compile_step = get_compile_step(project, build, env, config.parallel,
                                        config.upload_build_logs)
        build_steps.append(compile_step)
        if config.upload_build_logs:
          build_steps.append({
              'name':
                  'gcr.io/cloud-builders/gsutil',
              'args': [
                  '-m', 'cp', LOCAL_BUILD_LOG_PATH, config.upload_build_logs
              ],
          })

          # Report the build failure if it happened.
          build_steps.append({
              'name':
                  project.image,
              'args': [
                  'bash', '-c',
                  f'cat {LOCAL_BUILD_LOG_PATH} && test -f {BUILD_SUCCESS_MARKER}'
              ],
          })

        if project.run_tests:
          failure_msg = (
              '*' * 80 + '\\nBuild checks failed.\\n'
              'To reproduce, run:\\n'
              f'python infra/helper.py build_image {project.name}\\n'
              'python infra/helper.py build_fuzzers --sanitizer '
              f'{build.sanitizer} --engine {build.fuzzing_engine} '
              f'--architecture {build.architecture} {project.name}\\n'
              'python infra/helper.py check_build --sanitizer '
              f'{build.sanitizer} --engine {build.fuzzing_engine} '
              f'--architecture {build.architecture} {project.name}\\n' +
              '*' * 80)
          # Test fuzz targets.
          runner_image_name = build_lib.get_runner_image_name(
              config.test_image_suffix, config.base_image_tag)
          test_step = {
              'name': runner_image_name,
              'env': env,
              'args': [
                  'bash', '-c',
                  f'test_all.py || (echo "{failure_msg}" && false)'
              ],
              'id': get_id('build-check', build)
          }
          build_lib.dockerify_run_step(test_step, build)
          maybe_add_parallel(test_step, get_last_step_id(build_steps),
                             config.parallel)
          build_steps.append(test_step)

        if project.labels:
          # Write target labels.
          build_steps.append({
              'name':
                  project.image,
              'env':
                  env,
              'args': [
                  '/usr/local/bin/write_labels.py',
                  json.dumps(project.labels),
                  build.out,
              ],
          })

        build_steps.extend([
            # Generate targets list.
            {
                'name':
                    build_lib.get_runner_image_name(config.test_image_suffix,
                                                    config.base_image_tag),
                'env':
                    env,
                'args': [
                    'bash', '-c',
                    f'targets_list > /workspace/{build.targets_list_filename}'
                ],
            }
        ])
        if config.upload:
          upload_steps = get_upload_steps(project, build, timestamp,
                                          config.testing)
          build_steps.extend(upload_steps)

  return build_steps


def _indexer_built_image_name(name: str):
  # TODO(ochang): Write this to a tar (via docker image save) and upload this to
  # GCS.
  return f'us-docker.pkg.dev/oss-fuzz/indexer/{name}'


def _tracer_built_image_name(name: str):
  # TODO(ochang): Write this to a tar (via docker image save) and upload this to
  # GCS.
  return f'us-docker.pkg.dev/oss-fuzz/tracer/{name}'


def _create_indexed_build_steps(project,
                                build,
                                timestamp,
                                env,
                                indexer_targets,
                                build_type='indexer'):
  """Creates the build steps for a specific indexer type."""
  if build_type == 'indexer':
    container_name = _INDEXED_CONTAINER_NAME
    image_name = _indexer_built_image_name(project.name)
    build_script_command = '/opt/indexer/index_build.py'
  elif build_type == 'tracer':
    container_name = _TRACING_CONTAINER_NAME
    image_name = _tracer_built_image_name(project.name)
    build_script_command = (
        '/opt/indexer/index_build.py --tracing-instrumentation')
  else:
    raise ValueError(f'Unknown build_type: {build_type}')

  if indexer_targets:
    build_script_command += ' -t ' + ','.join(indexer_targets)

  command_sequence = [
      'cd /src',
      f'cd {project.workdir}',
      f'mkdir -p {build.out}',
      build_script_command,
      # Enable re-building both the project and the indexes.
      'cp -n /usr/local/bin/replay_build.sh $$SRC/',
      # The following complicated dance is so that the resulting image works
      # in a standalone way without the mount paths GCB installs by default
      # (/workspace).
      # First we save the $OUT contents into /outbak.
      'mkdir -p /outbak && mv $$OUT/* /outbak/ && '
      # Create a link from /out to the actual $OUT.
      'rm -rf /out && ln -s $$OUT /out && '
      # Unmount GCB's mount paths.
      'umount /workspace && mkdir -p $$OUT && '
      # Restore the actual contents into $OUT after unmounting.
      'mv -T /outbak $$OUT && '
      # Unshallow the main repository so we have easy access to the git history.
      f'/usr/local/bin/unshallow_repos.py {project.main_repo}',
  ]

  build_step = {
      'name': project.image,
      'args': ['bash', '-c', ' && '.join(command_sequence)],
      'env': env,
  }
  build_lib.dockerify_run_step(build_step,
                               build,
                               use_architecture_image_name=build.is_arm,
                               container_name=container_name)

  upload_steps = []
  # TODO: Don't upload anything if we're in trial build.
  if build_type == 'indexer':
    prefix = f'indexer_indexes/{project.name}/{timestamp}/'
    signed_policy_document = build_lib.get_signed_policy_document_upload_prefix(
        'clusterfuzz-builds', prefix)
    curl_signed_args = shlex.join(
        build_lib.signed_policy_document_curl_args(signed_policy_document))
    upload_steps = [
        {
            # TODO(metzman): Make sure not to include other tars, and support
            # .tar.gz
            'name': get_uploader_image(),
            'args': [
                '-c', f'for tar in {build.out}/*.tar; '
                f'do curl {curl_signed_args} -F key="{prefix}$(basename $tar)" '
                f'-F file="@$tar" https://{signed_policy_document.bucket}'
                '.storage.googleapis.com; '
                'done'
            ],
            'entrypoint': 'bash',
            'allowFailure': True,
        },
        build_lib.upload_using_signed_policy_document('/workspace/srcmap.json',
                                                      f'{prefix}srcmap.json',
                                                      signed_policy_document)
    ]

  push_image_steps = [
      {
          'name':
              build_lib.DOCKER_TOOL_IMAGE,
          'args': [
              'container', 'commit', '-c', 'ENV REPLAY_ENABLED 1',
              container_name, image_name + f':{timestamp}'
          ],
      },
      {
          'name': build_lib.DOCKER_TOOL_IMAGE,
          'args': ['tag', image_name + f':{timestamp}', image_name],
      },
      {
          'name': build_lib.DOCKER_TOOL_IMAGE,
          'args': ['push', '--all-tags', image_name],
      },
  ]
  return [build_step] + upload_steps + push_image_steps


def get_indexer_build_steps(project_name,
                            project_yaml,
                            dockerfile,
                            config,
                            additional_env=None,
                            use_caching=False,
                            timestamp=None):
  """Get indexer build steps."""
  project = Project(project_name, project_yaml, dockerfile)

  if project.fuzzing_language not in {'c', 'c++'}:
    return []

  if not timestamp:
    timestamp = get_datetime_now()
  timestamp = timestamp.strftime('%Y%m%d%H%M')

  build_steps = build_lib.get_project_image_steps(
      project.name,
      project.image,
      project.fuzzing_language,
      config=config,
      architectures=project.architectures,
      experiment=config.experiment)
  build = Build('none', 'address', 'x86_64')
  env = get_env(project.fuzzing_language, build, project.name)
  env.append('INDEXER_BUILD=1')
  env.append('CAPTURE_REPLAY_SCRIPT=1')

  indexer_steps = _create_indexed_build_steps(project,
                                              build,
                                              timestamp,
                                              env,
                                              project.indexer_targets,
                                              build_type='indexer')

  tracer_steps = _create_indexed_build_steps(project,
                                             build,
                                             timestamp,
                                             env,
                                             project.indexer_targets,
                                             build_type='tracer')
  return build_steps + indexer_steps + tracer_steps


def get_targets_list_upload_step(bucket, project, build, uploader_image):
  """Returns the step to upload targets_list for |build| of |project| to
  |bucket|."""
  targets_list_url = build_lib.get_signed_url(
      build_lib.get_targets_list_url(bucket, project.name, build.sanitizer))
  return {
      'name': uploader_image,
      'args': [
          f'/workspace/{build.targets_list_filename}',
          targets_list_url,
      ],
  }


def get_uploader_image():
  """Returns the uploader base image in |base_images_project|."""
  return f'gcr.io/{build_lib.BASE_IMAGES_PROJECT}/uploader'


def get_upload_steps(project, build, timestamp, testing):
  """Returns the steps for uploading the fuzzer build specified by |project| and
  |build|. Uses |timestamp| for naming the uploads. Uses |testing| for
  determining which image to use for the upload."""
  bucket = build_lib.get_upload_bucket(build.fuzzing_engine, build.architecture,
                                       testing)
  stamped_name = '-'.join([project.name, build.sanitizer, timestamp])
  zip_file = stamped_name + '.zip'
  upload_url = build_lib.get_signed_url(
      build_lib.GCS_UPLOAD_URL_FORMAT.format(bucket, project.name, zip_file))
  stamped_srcmap_file = stamped_name + '.srcmap.json'
  srcmap_url = build_lib.get_signed_url(
      build_lib.GCS_UPLOAD_URL_FORMAT.format(bucket, project.name,
                                             stamped_srcmap_file))
  latest_version_file = '-'.join(
      [project.name, build.sanitizer, LATEST_VERSION_FILENAME])
  latest_version_url = build_lib.GCS_UPLOAD_URL_FORMAT.format(
      bucket, project.name, latest_version_file)
  latest_version_url = build_lib.get_signed_url(
      latest_version_url, content_type=LATEST_VERSION_CONTENT_TYPE)
  uploader_image = get_uploader_image()

  upload_steps = [
      # Zip binaries.
      {
          'name': project.image,
          'args': ['bash', '-c', f'cd {build.out} && zip -r {zip_file} *'],
      },
      # Upload srcmap.
      {
          'name': uploader_image,
          'args': [
              '/workspace/srcmap.json',
              srcmap_url,
          ],
      },
      # Upload binaries.
      {
          'name': uploader_image,
          'args': [
              os.path.join(build.out, zip_file),
              upload_url,
          ],
      },
      # Upload targets list.
      get_targets_list_upload_step(bucket, project, build, uploader_image),
      # Upload the latest.version file.
      build_lib.http_upload_step(zip_file, latest_version_url,
                                 LATEST_VERSION_CONTENT_TYPE),
      # Cleanup.
      get_cleanup_step(project, build),
  ]
  return upload_steps


def get_cleanup_step(project, build):
  """Returns the step for cleaning up after doing |build| of |project|."""
  return {
      'name': project.image,
      'args': [
          'bash',
          '-c',
          'rm -r ' + build.out,
      ],
  }


# pylint: disable=no-member,too-many-arguments
def run_build(oss_fuzz_project,
              build_steps,
              credentials,
              build_type,
              cloud_project='oss-fuzz',
              extra_tags=None,
              experiment=False,
              timeout=None):
  """Run the build for given steps on cloud build. |build_steps| are the steps
  to run. |credentials| are are used to authenticate to GCB and build in
  |cloud_project|. |oss_fuzz_project| and |build_type| are used to tag the build
  in GCB so the build can be queried for debugging purposes."""
  if extra_tags is None:
    extra_tags = []
  tags = [oss_fuzz_project + '-' + build_type, build_type, oss_fuzz_project]
  tags.extend(extra_tags)
  if timeout is None:
    timeout = build_lib.BUILD_TIMEOUT
  bucket = GCB_LOGS_BUCKET if not experiment else GCB_EXPERIMENT_LOGS_BUCKET
  body_overrides = {
      'logsBucket': bucket,
      'queueTtl': str(QUEUE_TTL_SECONDS) + 's',
  }
  return build_lib.run_build(oss_fuzz_project,
                             build_steps,
                             credentials,
                             cloud_project,
                             timeout,
                             body_overrides=body_overrides,
                             tags=tags,
                             experiment=experiment)


def parse_args(description, args):
  """Parses command line arguments (or args if it is not None) and returns them.
  Suitable for a build script."""
  parser = argparse.ArgumentParser(sys.argv[0], description=description)
  parser.add_argument('projects', help='Projects.', nargs='+')
  parser.add_argument('--testing',
                      action='store_true',
                      required=False,
                      default=False,
                      help='Upload to testing buckets.')
  parser.add_argument('--test-image-suffix',
                      required=False,
                      default=None,
                      help='Use testing base-images.')
  parser.add_argument('--repo',
                      required=False,
                      default=DEFAULT_OSS_FUZZ_REPO,
                      help='Use specified OSS-Fuzz repo.')
  parser.add_argument('--branch',
                      required=False,
                      default=None,
                      help='Use specified OSS-Fuzz branch.')
  parser.add_argument('--parallel',
                      action='store_true',
                      required=False,
                      default=False,
                      help='Do builds in parallel.')
  parser.add_argument('--experiment',
                      action='store_true',
                      required=False,
                      default=False,
                      help='Configuration for experiments.')
  parser.add_argument('--fuzzing-engine',
                      required=False,
                      default='libfuzzer',
                      help='Fuzzing engine name.')
  parser.add_argument('--fuzz-target',
                      required=False,
                      default='',
                      help='Fuzz target name.')
  return parser.parse_args(args)


def create_config(args, build_type, base_image_tag=None):
  """Create a Config object from parsed command line |args|."""
  upload = not args.experiment
  return Config(testing=args.testing,
                test_image_suffix=args.test_image_suffix,
                base_image_tag=base_image_tag,
                branch=args.branch,
                parallel=args.parallel,
                upload=upload,
                experiment=args.experiment,
                build_type=build_type,
                fuzzing_engine=args.fuzzing_engine,
                fuzz_target=args.fuzz_target)


def build_script_main(script_description,
                      get_build_steps_func,
                      build_type,
                      args=None):
  """Gets arguments from command line using |script_description| as helpstring
  description or from args. Gets build_steps using |get_build_steps_func| and
  then runs those steps on GCB, tagging the builds with |build_type|. Returns 0
  on success, 1 on failure."""
  args = parse_args(script_description, args)
  logging.basicConfig(level=logging.INFO)

  credentials = oauth2client.client.GoogleCredentials.get_application_default()
  error = False
  for project_name in args.projects:
    try:
      project_yaml, dockerfile_contents = get_project_data(project_name)
    except FileNotFoundError:
      logging.error('Couldn\'t get project data. Skipping %s.', project_name)
      error = True
      continue

    base_image_tag = project_yaml.get('base_os_version', 'legacy')
    if base_image_tag == 'legacy':
      base_image_tag = None
    config = create_config(args, build_type, base_image_tag=base_image_tag)

    steps, _ = get_build_steps_func(project_name, project_yaml,
                                    dockerfile_contents, config)
    if not steps:
      logging.error('No steps. Skipping %s.', project_name)
      error = True
      continue

    run_build(project_name,
              steps,
              credentials,
              build_type,
              experiment=args.experiment)
  return 0 if not error else 1


def main():
  """Build and run projects."""
  return build_script_main('Builds a project on GCB.', get_build_steps,
                           FUZZING_BUILD_TYPE)


if __name__ == '__main__':
  sys.exit(main())