master.cfg

# -*- python -*-
# ex: set syntax=python:
# vim: set syntax=python:
# comment

import os
import re
from collections import defaultdict, namedtuple
from enum import Enum
from pathlib import Path

import buildbot.www.authz.endpointmatchers as ems
from buildbot.changes.filter import ChangeFilter
from buildbot.changes.gitpoller import GitPoller
from buildbot.config import BuilderConfig
from buildbot.locks import WorkerLock
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Interpolate, Property, renderer, Transform
from buildbot.reporters.generators.build import BuildStartEndStatusGenerator
from buildbot.reporters.github import GitHubStatusPush
from buildbot.reporters.message import MessageFormatterRenderable
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.timed import Nightly
from buildbot.steps.cmake import CMake
from buildbot.steps.master import SetProperties
from buildbot.steps.shell import SetPropertyFromCommand, ShellCommand
from buildbot.steps.source.github import GitHub
from buildbot.steps.transfer import FileUpload, FileDownload
from buildbot.steps.worker import MakeDirectory, SetPropertiesFromEnv, RemoveDirectory
from buildbot.worker import Worker
from buildbot.www.auth import UserPasswordAuth
from buildbot.www.authz import Authz
from buildbot.www.authz.roles import RolesFromUsername
from buildbot.www.hooks.github import GitHubEventHandler
from twisted.internet import defer

from custom_steps import CTest, CleanOldFiles, FileUploadIfNotExist, SetPropertiesFromCMakeCache

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

# SECRETS

GITHUB_TOKEN = Path('github_token.txt').read_text().strip()
WORKER_SECRET = Path('halide_bb_pass.txt').read_text().strip()
WEBHOOK_SECRET = Path('webhook_token.txt').read_text().strip()
WWW_PASSWORD = Path('buildbot_www_pass.txt').read_text().strip()

# SERVER SETTINGS

ARTIFACTS_DIR = os.environ.get('HALIDE_BB_MASTER_ARTIFACTS_DIR', '/home/halidenightly/artifacts')
REPO_DIR = Path(__file__, '..', '..').resolve()

# LLVM

# At any given time, we test (at least) 3 LLVM versions:
# - the current main (changes daily)
# - the most recent release (expected to be stable)
# - an older release (expected to be stable)
#
# The branches that correspond to these will rotate as new versions
# are released, but the underlying test logic should not need changing.

Version = namedtuple('Version', ['major', 'minor', 'patch'])
VersionedBranch = namedtuple('VersionedBranch', ['ref', 'version'])

LLVM_MAIN = 'main'
LLVM_RELEASE_18 = 'release_18'
LLVM_RELEASE_17 = 'release_17'
LLVM_RELEASE_16 = 'release_16'

LLVM_BRANCHES = {LLVM_MAIN: VersionedBranch(ref='main', version=Version(19, 0, 0)),
                 LLVM_RELEASE_18: VersionedBranch(ref='release/18.x', version=Version(18, 0, 0)),
                 LLVM_RELEASE_17: VersionedBranch(ref='llvmorg-17.0.6', version=Version(17, 0, 6)),
                 LLVM_RELEASE_16: VersionedBranch(ref='llvmorg-16.0.6', version=Version(16, 0, 6))}

# At any given time, Halide has a main branch, which supports (at least)
# the LLVM main branch and the most recent release branch (and maybe one older).
#
# We also support previous release branches; a release branch tracks *only* the
# corresponding version of LLVM (i.e., Halide 13 is 'release/13.x' and is only
# built/tested against LLVM13, even though it might still work with other LLVM versions).
#
# Note that we deliberately chose branch names that match LLVM's conventions.
#
# (Note that there are older releases of Halide that we no longer bother to build/test regularly.)

HALIDE_MAIN = 'main'
HALIDE_RELEASE_17 = 'release_17'
HALIDE_RELEASE_16 = 'release_16'

_HALIDE_RELEASES = [
    HALIDE_RELEASE_17,
    HALIDE_RELEASE_16,
]

HALIDE_BRANCHES = {HALIDE_MAIN: VersionedBranch(ref='main', version=Version(18, 0, 0)),
                   HALIDE_RELEASE_17: VersionedBranch(ref='release/17.x', version=Version(17, 0, 0)),
                   HALIDE_RELEASE_16: VersionedBranch(ref='release/16.x', version=Version(16, 0, 6))}

# This lists the Halide branch(es) for which we want to build nightlies;
# it's usually desirable to constrain these to save buildbot time (esp on the slower bots)
# and avoid branches that aren't changing much (i.e. -- recent releases that aren't
# likely to need new updates soon).
HALIDE_NIGHTLIES = [HALIDE_MAIN]

# Given a halide branch, return the 'native' llvm version we expect to use with it.
# For halide release branches, this is the corresponding llvm release branch; for
# halide main, it's llvm main.
LLVM_FOR_HALIDE = {
    HALIDE_MAIN: [LLVM_MAIN, LLVM_RELEASE_18, LLVM_RELEASE_17],
    HALIDE_RELEASE_17: [LLVM_RELEASE_17],
    HALIDE_RELEASE_16: [LLVM_RELEASE_16],
}

# WORKERS

# Can use Python 3.7 dataclasses instead, if we choose to upgrade to that.
WorkerConfig = namedtuple('WorkerConfig', ['max_builds', 'j', 'arch', 'bits', 'os'])

# Using nproc+2 on the arm32 builds causes internal errors in gcc-armeabihf. Let's just use nproc.
_NPROC = Interpolate("%(worker:numcpus)s")

# For machines with max_builds=1, using nproc+2 cores for building is the conventional choice
# (and what ninja defaults to). Oddly, "ninja -j 0" means "use as many threads as you like" which
# is definitely not what we want.
_NPROC_PLUS_2 = Transform(lambda x: f'{int(x) + 2}', _NPROC)

_WORKERS = [
    ('linux-worker-1', WorkerConfig(max_builds=4, j=8, arch='x86', bits=[32, 64], os='linux')),
    ('linux-worker-4', WorkerConfig(max_builds=4, j=8, arch='x86', bits=[32, 64], os='linux')),
    # Mac Mini 2018, 3.2 GHz 6-Core Intel Core i7, 16GB memory
    ('mac-x86-worker-2', WorkerConfig(max_builds=2, j=8, arch='x86', bits=[64], os='osx')),
    # Mac Mini 2018, ??? details TBD
    ('mac-x86-worker-3', WorkerConfig(max_builds=2, j=8, arch='x86', bits=[64], os='osx')),
    ('mac-arm-worker-1', WorkerConfig(max_builds=2, j=8, arch='arm', bits=[64], os='osx')),
    # The arm-linux bots here have 4 cores but apparently don't have enough RAM to do more
    # than -j=2 without crashing during LLVM builds.
    ('arm32-linux-worker-1', WorkerConfig(max_builds=1, j=2, arch='arm', bits=[32], os='linux')),
    ('arm32-linux-worker-2', WorkerConfig(max_builds=1, j=2, arch='arm', bits=[32], os='linux')),
    ('arm64-linux-worker-1', WorkerConfig(max_builds=1, j=2, arch='arm', bits=[64], os='linux')),
    ('arm64-linux-worker-2', WorkerConfig(max_builds=1, j=2, arch='arm', bits=[64], os='linux')),

    # The rpi4 has 8GB ram, so apparently it's OK with -j=nproc for now.
    # ('rpi4-linux-worker-1', WorkerConfig(max_builds=1, j=_NPROC, arch='arm', bits=[32], os='linux')),

    # OrangePi5+ test bots: 8 cores, can run 32 or 64
    ('arm64-linux-worker-3', WorkerConfig(max_builds=2, j=_NPROC, arch='arm', bits=[32, 64], os='linux')),
    ('arm64-linux-worker-4', WorkerConfig(max_builds=2, j=_NPROC, arch='arm', bits=[32, 64], os='linux')),

    ('win-worker-3', WorkerConfig(max_builds=2, j=8, arch='x86', bits=[32, 64], os='windows')),
    ('win-worker-4', WorkerConfig(max_builds=1, j=_NPROC_PLUS_2, arch='x86', bits=[32, 64], os='windows')),
]

# The 'workers' list defines the set of recognized buildworkers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.
c['workers'] = [Worker(n,
                       WORKER_SECRET,
                       keepalive_interval=300,  # default is 3600 (1 hour). We'll do 5 mins.
                       max_builds=cfg.max_builds,
                       properties={'WORKER_BUILD_PARALLELISM': cfg.j}) for n, cfg in _WORKERS]

_SANITIZERS = [
    'asan',
    'fuzzer',  # this isn't *technically* a sanitizer, but is close enough that it's a good fit
]

# LOCKS

# Performance testing requires exclusive use of a worker

# Compute-intensive build steps will grab this lock in reader
# mode. The performance test will grab it in exclusive mode.
performance_lock = WorkerLock("performance_lock", maxCount=9999)

# When building the LLVM nightlies, we can sync & build LLVM independently
# from other work, but when we update the install directory, we need to ensure
# we have an exclusive lock across the entire worker. (Since we have a small
# number of LLVM versions, and since 'make install' doesn't take very long,
# we could probably just get by with a single lock for *any* llvm install,
# but this isn't much harder to do.)
llvm_build_locks = {}
for llvm_branch, info in LLVM_BRANCHES.items():
    for bits in [32, 64]:
        llvm_build_locks[llvm_branch + str(bits)] = WorkerLock(
            f'llvm_install_lock_{info.version.major}_{bits}', maxCount=9999)

# CHANGESOURCES

# Here we point the buildbot at third-party codebases, ie. dependencies.
# Currently, we only have LLVM's `main` branch configured.

c['change_source'] = [
    GitPoller(
        repourl='https://github.com/llvm/llvm-project.git',
        workdir='gitpoller-llvm-workdir',
        branch=LLVM_BRANCHES[LLVM_MAIN].ref,
        pollInterval=60 * 60 * 24,  # Only check llvm once every 24 hours
        pollAtLaunch=True)
]

# CODEBASES

all_repositories = {
    'https://github.com/halide/Halide.git': 'halide',
    'https://github.com/llvm/llvm-project.git': 'llvm',
}


def codebase_generator(chdict):
    repo = chdict['repository']
    assert repo in all_repositories, "Codebase not found for chdict: " + str(chdict)
    return all_repositories[repo]


c['codebaseGenerator'] = codebase_generator


# BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them.  Note that any particular build will
# only take place on one worker.


class Purpose(Enum):
    halide_nightly = 1
    halide_testbranch = 2
    llvm_nightly = 3


class BuildSystem(Enum):
    make = 0
    cmake = 1


class BuilderType:
    """A class to encapsulate the settings for a specific Builder.
       (Do not confuse with CMake's 'BUILD_TYPE', which is something else.)

       It includes:
       - Halide 'target' in the form of arch-bits-os
       - LLVM branch to be used
       - CMake vs Make
       - halide-nightly vs halide-testbranch vs llvm-nightly
       - sanitizers vs none

       It doesn't currently include any 'features' because we don't currently
       bake any in at build time.

       It doesn't currently include the C++ compiler used (eg gcc7 vs gcc8 vs clang),
       mainly because we currently never test with multiple compilers for a given
       setup. (If we ever need to do so, compiler should be added to this.)
    """

    def __init__(self, arch, bits, os, halide_branch, llvm_branch, purpose, sanitizer=None,
                 buildsystem=BuildSystem.cmake):
        assert arch in ['arm', 'x86']
        assert bits in [32, 64]
        assert os in ['linux', 'windows', 'osx']
        assert llvm_branch in LLVM_BRANCHES, f'{llvm_branch} not recognized'

        self.arch = arch
        self.bits = bits
        self.os = os
        self.halide_branch = halide_branch
        self.llvm_branch = llvm_branch
        self.buildsystem = buildsystem
        self.purpose = purpose
        self.sanitizer = sanitizer

        if self.halide_branch:
            assert self.purpose != Purpose.llvm_nightly
            assert self.halide_branch in HALIDE_BRANCHES, f'unknown branch {self.halide_branch}'
            assert (self.purpose == Purpose.halide_testbranch or  # if not testbranch...
                    self.llvm_branch in LLVM_FOR_HALIDE[self.halide_branch])
        else:
            assert self.purpose == Purpose.llvm_nightly

        if self.sanitizer:
            assert self.sanitizer in _SANITIZERS

    # The armbots aren't configured with Python at all.
    # We don't support the Python bindings on 32-bit at all.
    def handles_python(self):
        if self.bits == 32:
            return False

        if self.arch == 'arm' and self.os == 'linux':
            return False

        if self.sanitizer_preset() is not None:
            return False

        return True

    # Only do the special build/test for testing serialization
    # via JIT on the x64-linux systems -- we could do them anywhere,
    # but on a single fast system is good enough
    def handles_serialization_jit_testing(self):
        if self.buildsystem != BuildSystem.cmake:
            return False

        return (self.arch == 'x86'
                and self.bits == 64
                and self.os == 'linux'
                and self.llvm_branch == LLVM_MAIN)

    def handles_sanitizers(self):
        if self.buildsystem != BuildSystem.cmake:
            return False

        return (self.arch == 'x86'
                and self.bits == 64
                and self.os == 'linux'
                and self.llvm_branch == LLVM_MAIN)

    def sanitizer_preset(self):
        if self.handles_sanitizers():
            if self.sanitizer == 'asan':
                return 'linux-x64-asan'
            if self.sanitizer == 'fuzzer':
                return 'linux-x64-fuzzer'

        return None

    def handles_riscv(self):
        # Only support RISCV on LLVM16 or later.
        return True

    def handles_hexagon(self):
        return (self.arch == 'x86'
                and self.bits == 64
                and self.os == 'linux'
                and self.llvm_branch == LLVM_MAIN)

    def handles_wasm(self):
        is_linux_x64 = (self.arch == 'x86'
                        and self.bits == 64
                        and self.os == 'linux')

        return (self.llvm_branch == LLVM_MAIN
                and (is_linux_x64 or self.os == 'osx'))

    def handles_wasm_wabt(self):
        return self.handles_wasm()

    def handles_wasm_v8(self):
        # OSX machines don't have V8 installed
        return self.handles_wasm() and self.os == 'linux'

    def has_nvidia(self):
        return (self.arch == 'x86'
                and self.bits == 64
                and self.os in ['windows', 'linux'])

    def handles_vulkan(self):
        # TODO: disabled temporarily pending fixes to the Vulkan runtime
        return False

        # Stick with Linux on x86-64 for now. Others TBD.
        # return (self.arch == 'x86'
        #         and self.bits == 64
        #         and self.os == 'linux'
        #         and self.halide_branch in [HALIDE_MAIN, HALIDE_RELEASE_16])

    def handles_webgpu(self):
        # At the moment, the WebGPU team recommends the OSX versions of Dawn/Node
        # as the most robust for testing, so that's all we're set up to test with.
        # (Note that 'Dawn' must be built/installed on the test machines manually;
        # there are no binaries/prebuilts available at this time.)
        return self.os == 'osx'

    def has_tflite(self):
        if self.arch == 'x86' and self.bits == 64 and self.os == 'linux':
            return True
        if self.arch == 'arm' and self.bits == 64 and self.os == 'osx':
            return True
        return False

    def has_ccache(self):
        return self.os in ['osx', 'linux']

    def halide_target(self):
        return '%s-%d-%s' % (self.arch, self.bits, self.os)

    def llvm_builder_label(self):
        return 'llvm-%s-%s' % (LLVM_BRANCHES[self.llvm_branch].version.major, self.halide_target())

    def halide_builder_label(self):
        # This currently tries to (somewhat) mimic the existing label pattern,
        # but is arbitrary. (If changed, manual purging of buildbot temporaries
        # is appropriate)
        a = ['halide']
        if self.sanitizer:
            a.append(self.sanitizer)
        if self.purpose == Purpose.halide_testbranch:
            a.append('testbranch')
        elif self.purpose == Purpose.halide_nightly:
            a.append('nightly')
        a.append(self.halide_branch)
        if self.halide_branch == HALIDE_MAIN:
            # Halide master is built against multiple LLVM versions,
            # so append that here for clarity
            a.append(f'llvm{LLVM_BRANCHES[self.llvm_branch].version.major}')
        a.append(self.halide_target())
        a.append(self.buildsystem.name)
        return '-'.join(a)

    def builder_label(self):
        if self.purpose == Purpose.llvm_nightly:
            return self.llvm_builder_label()
        else:
            return self.halide_builder_label()

    def builder_tags(self):
        return self.builder_label().split('-')

    def get_worker_names(self):
        return [n for n, cfg in _WORKERS
                if self.arch == cfg.arch and self.bits in cfg.bits and self.os == cfg.os]

    def __str__(self):
        return self.halide_target()


def get_builddir_subpath(subpath):
    # Normalize paths to use forward slashes.
    return Transform(lambda x: x.replace('\\', '/'), Interpolate(f'%(prop:builddir)s/{subpath}'))


def get_llvm_toolchains_path(*subpaths):
    return get_builddir_subpath(os.path.join('llvm-toolchains', *subpaths))


# TODO: make private to the LLVM code
def get_llvm_source_path(*subpaths):
    return get_builddir_subpath(os.path.join('llvm-project', *subpaths))


# TODO: make private to the LLVM code
def get_llvm_build_path(*subpaths):
    return get_builddir_subpath(os.path.join('llvm-build', *subpaths))


def get_llvm_install_path(builder_type, *subpaths):
    # Note that `builder_type.purpose` can be a Halide builder or an LLVM builder;
    # we want to ignore that aspect and produce the same effective path
    # regardless (ie, based only on the other aspects of the builder_type).
    llvm_workdir = builder_type.llvm_builder_label()
    return get_builddir_subpath(os.path.join('..', llvm_workdir, 'llvm-install', *subpaths))


def get_halide_source_path(*subpaths):
    return get_builddir_subpath(os.path.join('halide-source', *subpaths))


def get_halide_build_path(*subpaths):
    return get_builddir_subpath(os.path.join('halide-build', *subpaths))


def get_halide_install_path(builder_type, *subpaths):
    s = 'halide-install'
    if builder_type.sanitizer:
        s += '-' + builder_type.sanitizer
    return get_builddir_subpath(os.path.join(s, *subpaths))


def add_get_halide_source_steps(factory, builder_type):
    factory.addStep(GitHub(name='Get Halide source',
                           locks=[performance_lock.access('counting')],
                           codebase='halide',
                           workdir=get_halide_source_path(),
                           repourl='https://github.com/halide/Halide.git',
                           branch=HALIDE_BRANCHES[builder_type.halide_branch].ref,
                           mode='incremental'))


def add_get_llvm_source_steps(factory, builder_type):
    factory.addStep(GitHub(name=f'Get LLVM {LLVM_BRANCHES[builder_type.llvm_branch].version.major}',
                           locks=[performance_lock.access('counting')],
                           codebase='llvm',
                           workdir=get_llvm_source_path(),
                           repourl='https://github.com/llvm/llvm-project.git',
                           branch=LLVM_BRANCHES[builder_type.llvm_branch].ref,
                           mode='incremental'))

    # Always download the toolchains, even on platforms we don't need 'em
    toolchains_dir = get_llvm_toolchains_path()
    factory.addStep(MakeDirectory(name="Make CMake toolchain directory",
                                  locks=[performance_lock.access('counting')],
                                  dir=toolchains_dir,
                                  haltOnFailure=False))
    factory.addStep(FileDownload(name='Download CMake toolchains',
                                 mastersrc='toolchain.linux-arm32.cmake',  # relative to base dir
                                 workerdest='toolchain.linux-arm32.cmake',  # relative to workdir
                                 workdir=toolchains_dir,
                                 locks=[performance_lock.access('counting')],
                                 haltOnFailure=True,
                                 mode=0o644))


# Determined by running `set` in cmd.exe before and after vcvarsall.bat
# and diffing the output. It's likely that we don't need all of these
# to make things work, but I haven't bothered to figure out what is irrelevant,
# so I'm erring on the side of maybe too much.
# noinspection SpellCheckingInspection
VCVARSALL_ENV_VARS = [
    "COMMANDPROMPTTYPE",
    "DEVENVDIR",
    "EXTENSIONSDKDIR",
    "FRAMEWORK40VERSION",
    "FRAMEWORKDIR",
    "FRAMEWORKDIR64",
    "FRAMEWORKVERSION",
    "FRAMEWORKVERSION64",
    "INCLUDE",
    "LIB",
    "LIBPATH",
    "NETFXSDKDIR",
    "PATH",
    "PLATFORM",
    "UCRTVERSION",
    "UNIVERSALCRTSDKDIR",
    "VCIDEINSTALLDIR",
    "VCINSTALLDIR",
    "VCTOOLSINSTALLDIR",
    "VCTOOLSREDISTDIR",
    "VCTOOLSVERSION",
    "VISUALSTUDIOVERSION",
    "VS110COMNTOOLS",
    "VS120COMNTOOLS",
    "VS140COMNTOOLS",
    "VS160COMNTOOLS",
    "VSCMD_ARG_APP_PLAT",
    "VSCMD_ARG_HOST_ARCH",
    "VSCMD_ARG_TGT_ARCH",
    "VSCMD_VER",
    "VSINSTALLDIR",
    "WINDOWSLIBPATH",
    "WINDOWSSDKBINPATH",
    "WINDOWSSDKDIR",
    "WINDOWSSDKLIBVERSION",
    "WINDOWSSDKVERBINPATH",
    "WINDOWSSDKVERSION",
    "WINDOWSSDK_EXECUTABLEPATH_X64",
    "WINDOWSSDK_EXECUTABLEPATH_X86",
    "__DOTNET_ADD_64BIT",
    "__DOTNET_PREFERRED_BITNESS",
    "__VSCMD_PREINIT_PATH",
    "__VSCMD_SCRIPT_ERR_COUNT",
]


def get_msvc_config_steps(factory, builder_type):
    # ensure that we use the x64 host compiler, not the x86 host compiler
    arch_for_bits = {32: 'x64_x86', 64: 'x64'}
    vcvarsall = 'vcvarsall.bat %s && set' % arch_for_bits[builder_type.bits]

    # TODO: surely there is a better way of locating vcvarsall
    # vcvarsdir = "c:/Program Files (x86)/Microsoft Visual Studio/2019/Community/VC/Auxiliary/Build"
    vcvarsdir = "C:/Program Files/Microsoft Visual Studio/2022/Community/VC/Auxiliary/Build"

    # `vsvarsall && set` dumps all the settings to stdout;
    # we'll extract & save just the subset we think are likely to be relevant.
    def save_interesting_env_vars(rc, stdout, stderr):
        d = {}
        for line in stdout.split('\n'):
            match = re.match("^([a-zA-Z0-9_-]+)=(.*)$", line.strip())
            if match:
                key = match.group(1).upper()
                value = match.group(2)
                if key in VCVARSALL_ENV_VARS:
                    d[key] = value
        return {'env': d}

    factory.addStep(
        SetPropertyFromCommand(name='Run VcVarsAll',
                               description='Run VcVarsAll',
                               workdir=vcvarsdir,
                               locks=[performance_lock.access('counting')],
                               haltOnFailure=True,
                               command=vcvarsall,
                               extract_fn=save_interesting_env_vars))


def merge_renderable(_base, _extn):
    @renderer
    @defer.inlineCallbacks
    def render(props):
        base = yield props.render(_base)
        extn = yield props.render(_extn)
        base.update(extn)
        return base

    return render


def get_distrib_name(_version, target, ext):
    @renderer
    @defer.inlineCallbacks
    def render(props):
        rev = props.getProperty('got_revision')['halide']
        version = yield props.render(_version)
        return os.path.join(ARTIFACTS_DIR, f'Halide-{version}-{target}-{rev}.{ext}')

    return render


def get_cmake_generator(builder_type):
    return 'Ninja'


def get_llvm_cmake_options(builder_type):
    options = []
    return options


def get_halide_cmake_options(builder_type, build_dir):
    options = []

    if builder_type.sanitizer and builder_type.purpose != Purpose.llvm_nightly:
        assert builder_type.handles_sanitizers()
        options.append("--preset=%s" % builder_type.sanitizer_preset())
        # append *after* preset so we override the build dir
        options += ['-B', build_dir]

    return options


def get_ctest_options(builder_type, build_dir):
    assert builder_type.purpose != Purpose.llvm_nightly

    if builder_type.sanitizer:
        assert builder_type.handles_sanitizers()
        # No, this won't work, see https://gitlab.kitware.com/cmake/cmake/-/issues/23982 --
        # fortunately, we don't need to specify the current sanitizer toolchains
        # at test time (just at configure time).
        # return {'preset': builder_type.sanitizer_preset(), 'test_dir': build_dir}
        return {'build_config': builder_type.sanitizer_preset()}
    else:
        return {'build_config': 'Release'}


def get_halide_cmake_definitions(builder_type, halide_target='host', wasm_jit='wabt', extra_cmake_defs={}):
    cmake_definitions = {
        'Clang_DIR': get_llvm_install_path(builder_type, 'lib/cmake/clang'),
        'CMAKE_INSTALL_PREFIX': get_halide_install_path(builder_type),
        'Halide_TARGET': halide_target,
        'LLD_DIR': get_llvm_install_path(builder_type, 'lib/cmake/lld'),
        'LLVM_DIR': get_llvm_install_path(builder_type, 'lib/cmake/llvm'),
        'LLVM_ROOT': get_llvm_install_path(builder_type),
        'WITH_PYTHON_BINDINGS': 'ON' if builder_type.handles_python() else 'OFF',
        'WITH_TEST_FUZZ': 'ON' if builder_type.sanitizer == 'fuzzer' else 'OFF',
        **extra_cmake_defs
    }

    if builder_type.sanitizer and builder_type.handles_sanitizers():
        pass
    else:
        cmake_definitions['CMAKE_BUILD_TYPE'] = 'Release'

    # Sanitizer builds intermittently fail when using CCache for reasons that aren't
    # clear ("precompiled header modified") -- for now, just ignore CCache for them
    if builder_type.has_ccache() and not builder_type.sanitizer_preset():
        cmake_definitions['Halide_CCACHE_BUILD'] = 'ON'

    if builder_type.arch == 'arm' and builder_type.bits == 32 and builder_type.os == 'linux':
        # Halide always uses its own toolchain files, from the cmake/ subdir.
        cmake_definitions['CMAKE_TOOLCHAIN_FILE'] = get_halide_source_path('cmake', 'toolchain.linux-arm32.cmake')

    if builder_type.os == 'windows':
        cmake_definitions['CMAKE_TOOLCHAIN_FILE'] = Interpolate('%(prop:VCPKG_ROOT)s/scripts/buildsystems/vcpkg.cmake')
        # CMake on Windows can't reliably find our pip-installed PyBind11 unless we set CMAKE_PREFIX_PATH to point to is
        cmake_definitions['pybind11_DIR'] = Interpolate('%(prop:VIRTUAL_ENV)s/share/cmake/pybind11')

    # Don't bother with anything Python-related if we are targeting WebAssembly.
    if "wasm" in halide_target:
        cmake_definitions['WITH_PYTHON_BINDINGS'] = 'OFF'

    # TODO: HALIDE_NODE_JS_PATH is only necessary until EMSDK updates their built-in version of Node
    # to v16.13+; when that is done, remove this definition.
    if builder_type.handles_wasm() and halide_target.startswith("wasm-"):
        cmake_definitions['CMAKE_TOOLCHAIN_FILE'] = Interpolate(
            '%(prop:EMSDK)s/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake')
        cmake_definitions['NODE_JS_EXECUTABLE'] = Property('HALIDE_NODE_JS_PATH')
        if wasm_jit == 'v8':
            cmake_definitions['WITH_WABT'] = 'OFF'
            cmake_definitions['WITH_V8'] = 'ON'
            cmake_definitions['V8_INCLUDE_PATH'] = '/home/halidenightly/v8/v8/include'
            cmake_definitions['V8_LIB_PATH'] = \
                '/home/halidenightly/v8/v8/out/x64.release.static/obj/libv8_monolith.a'
        elif wasm_jit == 'wabt':
            cmake_definitions['WITH_WABT'] = 'ON'
            cmake_definitions['WITH_V8'] = 'OFF'
            cmake_definitions['V8_INCLUDE_PATH'] = ''
            cmake_definitions['V8_LIB_PATH'] = ''
        else:
            assert False, "Unknown wasm jit " + str(wasm_jit)

    if builder_type.handles_hexagon() and 'hvx' in halide_target:
        cmake_definitions['Halide_BUILD_HEXAGON_REMOTE_RUNTIME'] = 'ON'

    return cmake_definitions


def get_cmake_build_command(builder_type, build_dir, targets=None):
    cmd = ['ninja',
           '-C', build_dir,
           '-j', Property('WORKER_BUILD_PARALLELISM')]

    # TODO(srj): for debugging apps/c_backend
    if builder_type.os == "windows":
        cmd.append('-v')

    if targets:
        cmd.extend(targets)

    return cmd


# options in LLVM that we always want OFF because they aren't needed
# for Halide buildbot usage.
_LLVM_OFF_OPTS = [
    "CLANG_ENABLE_ARCMT"
    "CLANG_ENABLE_CLANGD"
    "CLANG_ENABLE_STATIC_ANALYZER"
    "CLANG_INCLUDE_DOCS"
    "CLANG_INSTALL_SCANBUILD"
    "CLANG_INSTALL_SCANVIEW"
    "CLANG_PLUGIN_SUPPORT"
    "CLANG_TIDY_ENABLE_STATIC_ANALYZER"
    "CLANG_TOOL_APINOTES_TEST_BUILD"
    "CLANG_TOOL_ARCMT_TEST_BUILD"
    "CLANG_TOOL_C_ARCMT_TEST_BUILD"
    "CLANG_TOOL_C_INDEX_TEST_BUILD"
    "CLANG_TOOL_CLANG_CHECK_BUILD"
    "CLANG_TOOL_CLANG_DIFF_BUILD"
    "CLANG_TOOL_CLANG_EXTDEF_MAPPING_BUILD"
    "CLANG_TOOL_CLANG_FORMAT_BUILD"
    "CLANG_TOOL_CLANG_FORMAT_VS_BUILD"
    "CLANG_TOOL_CLANG_FUZZER_BUILD"
    "CLANG_TOOL_CLANG_IMPORT_TEST_BUILD"
    "CLANG_TOOL_CLANG_LINKER_WRAPPER_BUILD"
    "CLANG_TOOL_CLANG_OFFLOAD_BUNDLER_BUILD"
    "CLANG_TOOL_CLANG_OFFLOAD_PACKAGER_BUILD"
    "CLANG_TOOL_CLANG_REFACTOR_BUILD"
    "CLANG_TOOL_CLANG_RENAME_BUILD"
    "CLANG_TOOL_CLANG_REPL_BUILD"
    "CLANG_TOOL_CLANG_SCAN_DEPS_BUILD"
    "CLANG_TOOL_CLANG_SHLIB_BUILD"
    "CLANG_TOOL_DIAGTOOL_BUILD"
    "CLANG_TOOL_DICTIONARY_BUILD"
    "CLANG_TOOL_LIBCLANG_BUILD"
    "CLANG_TOOL_NVPTX_ARCH_BUILD"
    "CLANG_TOOL_SCAN_BUILD_BUILD"
    "CLANG_TOOL_SCAN_BUILD_PY_BUILD"
    "CLANG_TOOL_SCAN_VIEW_BUILD"
    "CLANG_TOOLS_EXTRA_INCLUDE_DOCS"
    "LLVM_BUILD_32_BITS"
    "LLVM_BUILD_UTILS"
    "LLVM_ENABLE_BINDINGS"
    "LLVM_ENABLE_CURL"
    "LLVM_ENABLE_DIA_SDK"
    "LLVM_ENABLE_HTTPLIB"
    "LLVM_ENABLE_IDE"
    "LLVM_ENABLE_LIBXML2"
    "LLVM_ENABLE_OCAMLDOC"
    "LLVM_ENABLE_PLUGINS"
    "LLVM_ENABLE_TERMINFO"
    "LLVM_ENABLE_WARNINGS"
    "LLVM_ENABLE_ZSTD",
    "LLVM_INCLUDE_BENCHMARKS"
    "LLVM_INCLUDE_DOCS"
    "LLVM_INCLUDE_EXAMPLES"
    "LLVM_INCLUDE_TESTS"
    "LLVM_INCLUDE_UTILS"
    "LLVM_TOOL_BUGPOINT_BUILD"
    "LLVM_TOOL_BUGPOINT_PASSES_BUILD"
    "LLVM_TOOL_DSYMUTIL_BUILD"
    "LLVM_TOOL_DXIL_DIS_BUILD"
    "LLVM_TOOL_GOLD_BUILD"
    "LLVM_TOOL_LLC_BUILD"
    "LLVM_TOOL_LLI_BUILD"
    "LLVM_TOOL_LLVM_AR_BUILD"
    # "LLVM_TOOL_LLVM_AS_BUILD"  -- no: ClangConfig.cmake needs this
    # "LLVM_TOOL_LLVM_AS_FUZZER_BUILD"  -- no: ClangConfig.cmake needs this
    "LLVM_TOOL_LLVM_BCANALYZER_BUILD"
    "LLVM_TOOL_LLVM_C_TEST_BUILD"
    "LLVM_TOOL_LLVM_CAT_BUILD"
    "LLVM_TOOL_LLVM_CFI_VERIFY_BUILD"
    "LLVM_TOOL_LLVM_COV_BUILD"
    "LLVM_TOOL_LLVM_CVTRES_BUILD"
    "LLVM_TOOL_LLVM_CXXDUMP_BUILD"
    "LLVM_TOOL_LLVM_CXXFILT_BUILD"
    "LLVM_TOOL_LLVM_CXXMAP_BUILD"
    "LLVM_TOOL_LLVM_DEBUGINFO_ANALYZER_BUILD"
    "LLVM_TOOL_LLVM_DEBUGINFOD_BUILD"
    "LLVM_TOOL_LLVM_DEBUGINFOD_FIND_BUILD"
    "LLVM_TOOL_LLVM_DIFF_BUILD"
    "LLVM_TOOL_LLVM_DIS_BUILD"
    "LLVM_TOOL_LLVM_DIS_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_DLANG_DEMANGLE_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_DWARFDUMP_BUILD"
    "LLVM_TOOL_LLVM_DWARFUTIL_BUILD"
    "LLVM_TOOL_LLVM_DWP_BUILD"
    "LLVM_TOOL_LLVM_EXEGESIS_BUILD"
    "LLVM_TOOL_LLVM_EXTRACT_BUILD"
    "LLVM_TOOL_LLVM_GSYMUTIL_BUILD"
    "LLVM_TOOL_LLVM_IFS_BUILD"
    "LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_JITLINK_BUILD"
    "LLVM_TOOL_LLVM_JITLISTENER_BUILD"
    "LLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD"
    "LLVM_TOOL_LLVM_LINK_BUILD"
    "LLVM_TOOL_LLVM_LIPO_BUILD"
    "LLVM_TOOL_LLVM_LTO2_BUILD"
    "LLVM_TOOL_LLVM_LTO_BUILD"
    "LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_MC_BUILD"
    "LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_MCA_BUILD"
    "LLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_ML_BUILD"
    "LLVM_TOOL_LLVM_MODEXTRACT_BUILD"
    "LLVM_TOOL_LLVM_MT_BUILD"
    "LLVM_TOOL_LLVM_NM_BUILD"
    "LLVM_TOOL_LLVM_OBJCOPY_BUILD"
    "LLVM_TOOL_LLVM_OBJDUMP_BUILD"
    "LLVM_TOOL_LLVM_OPT_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_OPT_REPORT_BUILD"
    "LLVM_TOOL_LLVM_PDBUTIL_BUILD"
    "LLVM_TOOL_LLVM_PROFDATA_BUILD"
    "LLVM_TOOL_LLVM_PROFGEN_BUILD"
    "LLVM_TOOL_LLVM_RC_BUILD"
    "LLVM_TOOL_LLVM_READOBJ_BUILD"
    "LLVM_TOOL_LLVM_READTAPI_BUILD"
    "LLVM_TOOL_LLVM_REDUCE_BUILD"
    "LLVM_TOOL_LLVM_REMARKUTIL_BUILD"
    "LLVM_TOOL_LLVM_RTDYLD_BUILD"
    "LLVM_TOOL_LLVM_RUST_DEMANGLE_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_SHLIB_BUILD"
    "LLVM_TOOL_LLVM_SIM_BUILD"
    "LLVM_TOOL_LLVM_SIZE_BUILD"
    "LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_SPLIT_BUILD"
    "LLVM_TOOL_LLVM_STRESS_BUILD"
    "LLVM_TOOL_LLVM_STRINGS_BUILD"
    "LLVM_TOOL_LLVM_SYMBOLIZER_BUILD"
    "LLVM_TOOL_LLVM_TLI_CHECKER_BUILD"
    "LLVM_TOOL_LLVM_UNDNAME_BUILD"
    "LLVM_TOOL_LLVM_XRAY_BUILD"
    "LLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD"
    "LLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD"
    "LLVM_TOOL_LTO_BUILD"
    "LLVM_TOOL_MLIR_BUILD"
    "LLVM_TOOL_OBJ2YAML_BUILD"
    "LLVM_TOOL_OPENMP_BUILD"
    "LLVM_TOOL_OPT_BUILD"
    "LLVM_TOOL_OPT_VIEWER_BUILD"
    "LLVM_TOOL_POLLY_BUILD"
    "LLVM_TOOL_PSTL_BUILD"
    "LLVM_TOOL_REMARKS_SHLIB_BUILD"
    "LLVM_TOOL_SANCOV_BUILD"
    "LLVM_TOOL_SANSTATS_BUILD"
    "LLVM_TOOL_SPIRV_TOOLS_BUILD"
    "LLVM_TOOL_VERIFY_USELISTORDER_BUILD"
    "LLVM_TOOL_VFABI_DEMANGLE_FUZZER_BUILD"
    "LLVM_TOOL_XCODE_TOOLCHAIN_BUILD"
    "LLVM_TOOL_YAML2OBJ_BUILD"
]


def get_llvm_cmake_definitions(builder_type):
    # Keep sorted!
    definitions = {
        'CMAKE_BUILD_TYPE': 'Release',
        'CMAKE_INSTALL_PREFIX': get_llvm_install_path(builder_type),
        'LLVM_BUILD_32_BITS': ('ON' if builder_type.bits == 32 else 'OFF'),
        'LLVM_ENABLE_ASSERTIONS': 'ON',
        'LLVM_ENABLE_RTTI': 'ON',
        'LLVM_ENABLE_ZLIB': 'ON',
        'LLVM_TARGETS_TO_BUILD': 'X86;ARM;NVPTX;AArch64;Hexagon;PowerPC;WebAssembly',
    }

    for off in _LLVM_OFF_OPTS:
        definitions[off] = 'OFF'

    if builder_type.bits == 32:
        definitions['CMAKE_FIND_ROOT_PATH_MODE_INCLUDE'] = "ONLY"
        definitions['CMAKE_FIND_ROOT_PATH_MODE_LIBRARY'] = "ONLY"
        definitions['CMAKE_FIND_ROOT_PATH_MODE_PACKAGE'] = "ONLY"
        definitions['CMAKE_FIND_ROOT_PATH_MODE_PROGRAM'] = "NEVER"

    if builder_type.handles_riscv():
        definitions['LLVM_TARGETS_TO_BUILD'] += ";RISCV"

    if builder_type.handles_sanitizers():
        definitions['LLVM_ENABLE_RUNTIMES'] = "compiler-rt;libcxx;libcxxabi;libunwind"
        # We only need clang-tools-extra if building for sanitizers -- skip them
        # if the builder will never do this, to save time & space.
        definitions['LLVM_ENABLE_PROJECTS'] = "clang;lld;clang-tools-extra"
    else:
        definitions['LLVM_ENABLE_PROJECTS'] = "clang;lld"

    # Some versions of GCC will flood the output with useless warnings about
    # "parameter passing for argument of type foo changed in GCC 7.1" unless
    # we disable this warning. This isn't *essential*, but it makes looking at the
    # LLVM build logs much less noisy.
    if builder_type.os != 'windows':
        definitions['CMAKE_CXX_FLAGS'] = '-Wno-psabi'

    if builder_type.arch == 'arm' and builder_type.bits == 32 and builder_type.os == 'linux':
        # LLVM doesn't provide a toolchain file, and we can't/don't-want-to rely on the
        # one from Halide, so we'll rely on one that the buildbot downloads to each worker.
        # (Note that this assumes the file has been properly downloaded.)
        definitions['CMAKE_TOOLCHAIN_FILE'] = get_llvm_toolchains_path('toolchain.linux-arm32.cmake')
        definitions['LLVM_TARGET_ARCH'] = 'ARM'
        definitions['LLVM_DEFAULT_TARGET_TRIPLE'] = 'arm-linux-gnueabihf'

    if builder_type.arch == 'x86' and builder_type.bits == 32 and builder_type.os == 'linux':
        definitions['CMAKE_FIND_ROOT_PATH'] = '/usr/lib/i386-linux-gnu'
        definitions['CMAKE_FIND_ROOT_PATH_MODE_LIBRARY'] = 'ONLY'

    # This disables an XCode setting that can get enabled by default
    # when assertions are enabled, but only if your XCode install has
    # certain frameworks installed; we want it disabled, as it prevents
    # prebuilt libraries from working properly with XCode 9.x.
    if builder_type.os == 'osx':
        definitions['LLVM_ENABLE_SUPPORT_XCODE_SIGNPOSTS'] = 'FORCE_OFF'

    # We never build LLVM with sanitizers enabled
    if builder_type.has_ccache():
        # https://discourse.llvm.org/t/llvm-ccache-build-is-deprecated/68431/2
        # (Don't also set LLVM_CCACHE_BUILD, it will cause build failures when
        # combined with the code below on Linux)
        definitions['CMAKE_C_COMPILER_LAUNCHER'] = 'ccache'
        definitions['CMAKE_CXX_COMPILER_LAUNCHER'] = 'ccache'

    return definitions


def extend_property(dict_name, **kwargs):
    @renderer
    def render(props):
        table = props.getProperty(dict_name, default={})
        table.update(kwargs)
        return table

    return render


def add_env_setup_step(factory, builder_type, enable_ccache=False):
    if builder_type.os == 'windows':
        # do this first because the SetPropertyFromCommand step isn't smart enough to merge
        get_msvc_config_steps(factory, builder_type)

    cxx = 'c++'
    cc = 'cc'
    ld = 'ld'

    if builder_type.os == 'linux':
        cc = 'gcc-9'
        cxx = 'g++-9'
        ld = 'ld'
        if builder_type.arch == 'x86' and builder_type.bits == 32:
            cxx += ' -m32'
            cc += ' -m32'
            ld += ' -melf_i386'
    elif builder_type.os == 'windows':
        cxx = 'cl.exe'
        cc = 'cl.exe'

    # This is only necessary (or desirable) for make-based builds of Halide;
    # CMake-based builds handle it via Halide_CCACHE_BUILD and/or LLVM_CCACHE_BUILD
    if enable_ccache and builder_type.has_ccache():
        cxx = 'ccache ' + cxx
        cc = 'ccache ' + cc

    env = {
        'CC': cc,
        'CXX': cxx,
        'LD': ld,
    }

    # TODO: HALIDE_NODE_JS_PATH is only necessary until EMSDK updates their built-in version of Node
    # to v16.13+; when that is done, remove HALIDE_NODE_JS_PATH here and on the workers.
    factory.addStep(SetPropertiesFromEnv(name='Read worker environment',
                                         variables=['EMSDK',
                                                    'HALIDE_NODE_JS_PATH',
                                                    'HL_HEXAGON_TOOLS',
                                                    'HL_WEBGPU_NATIVE_LIB',
                                                    'HL_WEBGPU_NODE_BINDINGS',
                                                    'LD_LIBRARY_PATH',
                                                    'VIRTUAL_ENV',
                                                    'VCPKG_ROOT']))

    vcpkg_root = Property('VCPKG_ROOT', default=None)

    if builder_type.handles_hexagon():
        # Environment variables for testing Hexagon DSP
        hexagon_remote_bin = get_halide_source_path('src', 'runtime', 'hexagon_remote', 'bin')

        # Assume that HL_HEXAGON_TOOLS points to the correct directory (it might not be /usr/local/hexagon)
        env['HL_HEXAGON_SIM_REMOTE'] = Transform(os.path.join, hexagon_remote_bin, 'v65', 'hexagon_sim_remote')
        env['HL_HEXAGON_SIM_CYCLES'] = '1'
        env['LD_LIBRARY_PATH'] = [
            # no, this will cause a failure at runtime if LD_LIBRARY_PATH is unset (or empty!)
            # Property('LD_LIBRARY_PATH'),
            Transform(os.path.join, hexagon_remote_bin, 'host'),
            Interpolate('%(prop:HL_HEXAGON_TOOLS)s/lib/iss'),
        ]
        env['HEXAGON_SDK_ROOT'] = Interpolate('%(prop:HL_HEXAGON_TOOLS)s/../../../..')

    # Force Vulkan validation layer on to catch any driver related errors
    # ... this enables a suite of diagnostic checks implemented in the Vulkan SDK
    # that verifies the driver and application conform to the Vulkan runtime
    # specification. This should not be enabled in production due to the overhead,
    # but we want to catch any changes in driver behaviour and/or spurious errors that
    # may be hard to find (but easy to fix if the right error messages are present)
    if builder_type.has_nvidia() and builder_type.handles_vulkan():
        env['VK_INSTANCE_LAYERS'] = "VK_LAYER_KHRONOS_validation"

    if builder_type.os == 'osx':
        # Environment variable for turning on Metal API validation
        # This will have no effect on CPU testing, just Metal testing
        env['METAL_DEVICE_WRAPPER_TYPE'] = '1'

    if builder_type.os == 'windows':
        vcpkg_root = Property('VCPKG_ROOT', default='C:/vcpkg')
        env['VCPKG_ROOT'] = vcpkg_root

        # Current NVidia drivers on our Windows buildbots can corrupt their own
        # cache, leading to many spurious failures. Disable the cache
        # for now, pending NVidia investigation.
        env['CUDA_CACHE_DISABLE'] = '1'

        # We don't ever want an Abort, Retry, Ignore dialog in our tests
        env['HL_DISABLE_WINDOWS_ABORT_DIALOG'] = '1'

    # Leaving this here (but commented out) in case we need to temporarily
    # disable leak-checking in the future.
    #
    # if builder_type.handles_sanitizers():
    #     # Disable leak-checking (for now) for ASAN builds
    #     env['ASAN_OPTIONS'] = 'detect_leaks=0'

    factory.addStep(SetProperties(
        name='Initialize environment',
        properties=dict(
            env=extend_property('env', **env),
            VCPKG_ROOT=vcpkg_root)))


def add_llvm_steps(factory, builder_type, clean_rebuild):
    build_dir = get_llvm_build_path()
    install_dir = get_llvm_install_path(builder_type)
    llvm_name = str(LLVM_BRANCHES[builder_type.llvm_branch].version.major)

    if clean_rebuild:
        factory.addStep(RemoveDirectory(name="Remove LLVM %s Build Dir" % llvm_name,
                                        locks=[performance_lock.access('counting')],
                                        dir=build_dir,
                                        haltOnFailure=False))
        factory.addStep(RemoveDirectory(name="Remove LLVM %s Install Dir" % llvm_name,
                                        locks=[performance_lock.access('counting')],
                                        dir=install_dir,
                                        haltOnFailure=False))

    factory.addStep(MakeDirectory(name="Make LLVM %s Build Dir" % llvm_name,
                                  locks=[performance_lock.access('counting')],
                                  dir=build_dir,
                                  haltOnFailure=False))
    factory.addStep(MakeDirectory(name="Make LLVM %s Install Dir" % llvm_name,
                                  locks=[performance_lock.access('counting')],
                                  dir=install_dir,
                                  haltOnFailure=False))

    factory.addStep(
        CMake(name='Configure LLVM %s' % llvm_name,
              locks=[performance_lock.access('counting')],
              haltOnFailure=True,
              env=Property('env'),
              workdir=build_dir,
              path=get_llvm_source_path('llvm'),
              generator=get_cmake_generator(builder_type),
              definitions=get_llvm_cmake_definitions(builder_type),
              options=get_llvm_cmake_options(builder_type)))

    factory.addStep(
        ShellCommand(name='Build LLVM %s' % llvm_name,
                     locks=[performance_lock.access('counting')],
                     haltOnFailure=True,
                     workdir=build_dir,
                     env=Property('env'),
                     command=get_cmake_build_command(builder_type, build_dir, targets=['install'])))

    # Save the SHA of LLVM's head rev into ${INSTALL}/llvm_version.txt,
    # just to make debugging simpler
    def get_llvm_latest_commit_cmd(builder_type):
        @renderer
        @defer.inlineCallbacks
        def render(props):
            rev = props.getProperty('got_revision')['llvm']
            _llvm_install_path = get_llvm_install_path(builder_type)
            llvm_install_path = yield props.render(_llvm_install_path)
            return f'echo {rev} > {llvm_install_path}/llvm_latest_commit.txt'

        return render

    factory.addStep(
        ShellCommand(name='Stamp Install Directory for LLVM %s' % llvm_name,
                     locks=[performance_lock.access('counting')],
                     haltOnFailure=True,
                     workdir=get_llvm_source_path(),
                     env=Property('env'),
                     command=get_llvm_latest_commit_cmd(builder_type)))

    # Note that this must only be called by a Renderer
    def get_packaged_name(props, builder_type):
        v = LLVM_BRANCHES[builder_type.llvm_branch].version
        version = "%d.%d.%d" % (v.major, v.minor, v.patch)
        # 'halide_target()' is correct here because we want the halide-style
        # triple that this llvm was build for
        target = builder_type.halide_target()
        rev = props.getProperty('got_revision')['llvm']
        ext = 'zst'
        return f'llvm-{version}-{target}-{rev}.{ext}'

    def get_local_packaged_dir():
        return get_builddir_subpath('llvm_packaged')

    def make_local_packaged_llvm_cmd(builder_type):
        @renderer
        @defer.inlineCallbacks
        def render(props):
            _local_packaged_dir = get_local_packaged_dir()
            local_packaged_dir = yield props.render(_local_packaged_dir)
            packaged_name = get_packaged_name(props, builder_type)
            local_packaged_path = os.path.join(local_packaged_dir, packaged_name)
            # Use zst compression rather than zip/gzip, as it's *much*
            # faster (seconds instead of minutes); it's not as broadly supported,
            # but we only want this for internal use, so that's fine.
            cmd = ['cmake', '-E', 'tar', 'cv',
                   local_packaged_path,
                   '--zstd',
                   '.']
            return cmd

        return render

    factory.addStep(
        MakeDirectory(name="Make LLVM Package Directory",
                      locks=[performance_lock.access('counting')],
                      dir=get_local_packaged_dir(),
                      haltOnFailure=True))

    factory.addStep(
        ShellCommand(name='Package LLVM',
                     description='Package LLVM',
                     # It's critical we set this as the working dir to get relatize
                     # paths in the resulting archive.
                     workdir=get_llvm_install_path(builder_type),
                     locks=[performance_lock.access('counting')],
                     haltOnFailure=True,
                     command=make_local_packaged_llvm_cmd(builder_type)))

    def get_local_packaged_path_r(builder_type):
        @renderer
        @defer.inlineCallbacks
        def render(props):
            _local_packaged_dir = get_local_packaged_dir()
            local_packaged_dir = yield props.render(_local_packaged_dir)
            packaged_name = get_packaged_name(props, builder_type)
            local_packaged_path = os.path.join(local_packaged_dir, packaged_name)
            return local_packaged_path

        return render

    def get_upload_dest_path(builder_type):
        @renderer
        @defer.inlineCallbacks
        def render(props):
            p = yield os.path.join(ARTIFACTS_DIR, get_packaged_name(props, builder_type))
            return p

        return render

    factory.addStep(
        FileUploadIfNotExist(name='Upload LLVM package',
                             workersrc=get_local_packaged_path_r(builder_type),
                             locks=[performance_lock.access('counting')],
                             haltOnFailure=True,
                             workdir=build_dir,
                             mode=0o644,
                             masterdest=get_upload_dest_path(builder_type)))

    def pkg_version_and_target(path: Path):
        # Archives names are formatted like: llvm-[version]-[arch]-[commit].[ext]
        # This grabs "llvm-[version]-[arch]".
        match = re.match(r'^(.*)-[a-f0-9]+\.zst', path.name)
        return match.group(1) if match else None

    factory.addStep(CleanOldFiles(
        name='Clean old LLVM packages',
        workdir=ARTIFACTS_DIR,
        locks=[performance_lock.access('counting')],
        groupfn=pkg_version_and_target))


def add_halide_cmake_build_steps(factory, builder_type):
    # Always do a clean build for Halide
    source_dir = get_halide_source_path()
    build_dir = get_halide_build_path()
    install_dir = get_halide_install_path(builder_type)
    factory.addStep(RemoveDirectory(name="Remove Halide Build Dir",
                                    locks=[performance_lock.access('counting')],
                                    dir=build_dir,
                                    haltOnFailure=False))
    factory.addStep(MakeDirectory(name="Make Halide Build Dir",
                                  locks=[performance_lock.access('counting')],
                                  dir=build_dir,
                                  haltOnFailure=False))

    factory.addStep(RemoveDirectory(name="Remove Halide Install Dir",
                                    locks=[performance_lock.access('counting')],
                                    dir=install_dir,
                                    haltOnFailure=False))
    factory.addStep(MakeDirectory(name="Make Halide Install Dir",
                                  locks=[performance_lock.access('counting')],
                                  dir=install_dir,
                                  haltOnFailure=False))

    factory.addStep(CMake(name='Configure Halide',
                          description='Configure Halide',
                          locks=[performance_lock.access('counting')],
                          haltOnFailure=True,
                          workdir=build_dir,
                          env=Property('env'),
                          path=source_dir,
                          generator=get_cmake_generator(builder_type),
                          definitions=get_halide_cmake_definitions(builder_type),
                          options=get_halide_cmake_options(builder_type, build_dir)))

    factory.addStep(
        ShellCommand(name='Build Halide',
                     description='Build Halide',
                     locks=[performance_lock.access('counting')],
                     haltOnFailure=True,
                     workdir=build_dir,
                     env=Property('env'),
                     command=get_cmake_build_command(builder_type, build_dir, targets=['all', 'install'])))


def add_halide_cmake_package_steps(factory, builder_type):
    source_dir = get_halide_source_path()

    target = builder_type.halide_target()
    ext = 'zip' if builder_type.os == 'windows' else 'tar.gz'

    factory.addStep(
        SetPropertiesFromCMakeCache(
            name='Get Halide package version',
            workdir=get_halide_build_path(),
            props=['CMAKE_PROJECT_VERSION']))

    extra_env = dict(
        Clang_DIR=get_llvm_install_path(builder_type, 'lib/cmake/clang'),
        LLD_DIR=get_llvm_install_path(builder_type, 'lib/cmake/lld'),
        LLVM_DIR=get_llvm_install_path(builder_type, 'lib/cmake/llvm'),
        Halide_VERSION=Property('CMAKE_PROJECT_VERSION')
    )

    if builder_type.os == 'windows':
        # TODO: on Windows, we can't use Ninja for packaging (as we do everywhere
        # else in this cfg) due to a bug in CMake 3.18, so we must use MSBuild;
        # that means we must use a different build directory entirely. To simplify the
        # world, we make this a subdir of the real build dir (so it gets cleaned properly).
        # https://github.com/halide/Halide/issues/5264
        build_dir = get_halide_build_path("packaging_dir")

        if builder_type.arch == 'arm':
            arch = 'ARM' if builder_type.bits == 32 else 'ARM64'
        else:
            arch = 'Win32' if builder_type.bits == 32 else 'x64'
        cmd = [get_halide_source_path('packaging/zip/package.bat'), source_dir, build_dir, arch]
    else:
        build_dir = get_halide_build_path()
        cmd = [get_halide_source_path('packaging/tgz/package.sh'), source_dir, build_dir]
        if builder_type.arch == 'arm' and builder_type.bits == 32 and builder_type.os == 'linux':
            extra_env['CMAKE_TOOLCHAIN_FILE'] = get_halide_source_path('cmake', 'toolchain.linux-arm32.cmake')

    factory.addStep(
        ShellCommand(name='Package Halide',
                     description='Package Halide',
                     workdir=build_dir,
                     env=extend_property('env', **extra_env),
                     locks=[performance_lock.access('counting')],
                     haltOnFailure=True,
                     command=cmd))

    factory.addStep(
        FileUpload(name='Upload Halide package',
                   workersrc=Interpolate(f'Halide-%(prop:CMAKE_PROJECT_VERSION)s-{target}.{ext}'),
                   locks=[performance_lock.access('counting')],
                   haltOnFailure=True,
                   workdir=build_dir,
                   mode=0o644,
                   masterdest=get_distrib_name(Property('CMAKE_PROJECT_VERSION'), target, ext)))

    def pkg_version_and_target(path: Path):
        # Archives names are formatted like: Halide-[version]-[arch]-[commit].[ext]
        # This grabs "Halide-[version]-[arch]".
        match = re.match(r'^(.*)-[a-f0-9]+\.(tar\.gz|tgz|zip)', path.name)
        return match.group(1) if match else None

    factory.addStep(CleanOldFiles(
        name='Clean old Halide releases',
        workdir=ARTIFACTS_DIR,
        locks=[performance_lock.access('counting')],
        groupfn=pkg_version_and_target))


# Figure out which "non-cpu" (GPU, DSP, etc) targets this builder can handle.
# Return (target, is_simulator)
def get_gpu_dsp_targets(builder_type):
    if builder_type.sanitizer_preset() is not None:
        return

    if builder_type.has_nvidia():
        yield 'host-cuda', False
        yield 'host-opencl', False

        # TODO: temporarily disabled because our only windows bot doesn't support it...
        # if builder_type.os == 'windows':
        #     yield 'host-d3d12compute', False

        # If we're running on a capable GPU, add all optional feature flags to the vulkan target
        # which are required to get all the correctness tests to pass
        if builder_type.handles_vulkan():
            yield 'host-vulkan-vk_int8-vk_int16-vk_int64-vk_float16-vk_float64-vk_v13', False

    if builder_type.handles_webgpu():
        yield 'host-webgpu', False

    if builder_type.os == 'osx':
        yield 'host-metal', False

    if builder_type.handles_hexagon():
        # All the buildbots use a simulator for HVX, so performance tests
        # won't be useful
        yield 'host-hvx', True


# Return a dict with halide-targets as the keys, and a list of test-labels for each value.
def get_test_labels(builder_type):
    targets = defaultdict(list)

    preset = builder_type.sanitizer_preset()

    # For the fuzz sanitizer, run only the fuzz tests
    if preset and 'fuzz' in preset:
        targets['host'].extend(['fuzz'])
        return targets

    targets['host'].extend(['internal', 'correctness', 'generator',
                            'autoschedulers_cpu', 'error', 'warning', 'apps', 'performance', 'tutorial'])

    # For all other sanitizers (eg asan), don't bother with the gpu/etc tests.
    if preset:
        return targets

    # TODO: some JIT+generator tests are failing on arm32; disable for now
    # pending fixes (see https://github.com/halide/Halide/issues/4940)
    if builder_type.arch == 'arm' and builder_type.bits == 32 and builder_type.os == 'linux':
        targets['host'].remove('internal')
        targets['host'].remove('generator')

    if builder_type.handles_python():
        targets['host'].extend(['python'])

    # Test without SSE4.1 on all x86 systems
    if builder_type.arch == 'x86':
        t = 'x86-%d-%s' % (builder_type.bits, builder_type.os)
        targets[t].extend(['correctness'])
        # on x86-64, also test with SSE4.1 (but nothing else that 'host' might sniff)
        if builder_type.bits == 64:
            targets['%s-sse41' % t].extend(['correctness'])

    # Test a subset of things on GPU/DSP targets, as appropriate
    for t, is_simulator in get_gpu_dsp_targets(builder_type):
        # TODO(https://github.com/halide/Halide/issues/7420): disable apps for host-gpu until the errors are resolved
        if t == 'host-webgpu':
            targets[t].extend(['correctness', 'generator'])
        else:
            targets[t].extend(['correctness', 'generator', 'apps'])
        if 'cuda' in t:
            targets[t].extend(['autoschedulers_cuda'])
        if 'hvx' not in t:
            targets[t].extend(['autoschedulers_gpu'])
        # Don't do performance testing on simulators.
        if not is_simulator:
            targets[t].extend(['performance'])

    # Handle this special case separately
    if builder_type.has_nvidia():
        targets['host-cuda-opencl'].extend(['correctness_multi_gpu'])

        # If we're running on a capable GPU, add all optional feature flags to the vulkan target
        # which are required to get all the correctness tests to pass
        if builder_type.handles_vulkan():
            targets['host-vulkan-vk_int8-vk_int16-vk_int64-vk_float16-vk_float64-vk_v13'].extend(
                ['internal', 'correctness', 'generator', 'error', 'warning'])

    if builder_type.handles_wasm():
        if HALIDE_BRANCHES[builder_type.halide_branch].version.major < 17:
            if builder_type.handles_wasm_wabt():
                # TODO: this is a horrid hack. For now, we want to test JIT with both WABT and V8.
                # Add as a horrible wart on the target string.
                targets['wasm-32-wasmrt-wasm_simd128-wasm_signext-wasm_sat_float_to_int/wabt'].extend(
                    ['internal', 'correctness', 'generator', 'error', 'warning'])

            if builder_type.handles_wasm_v8():
                # TODO: this is a horrid hack. For now, we want to test JIT with both WABT and V8.
                # Add as a horrible wart on the target string.
                targets['wasm-32-wasmrt-wasm_simd128-wasm_signext-wasm_sat_float_to_int/v8'].extend(
                    ['internal', 'correctness', 'generator', 'error', 'warning'])

            # WABT (and thus WASM JIT) can't handle code build with wasm_threads yet,
            # so only test Generator here
            targets['wasm-32-wasmrt-wasm_simd128-wasm_signext-wasm_sat_float_to_int-wasm_threads'].extend(
                ['generator', 'apps'])
        else:
            if builder_type.handles_wasm_wabt():
                # TODO: this is a horrid hack. For now, we want to test JIT with both WABT and V8.
                # Add as a horrible wart on the target string.
                targets['wasm-32-wasmrt-wasm_simd128/wabt'].extend(
                    ['internal', 'correctness', 'generator', 'error', 'warning'])

                # Do at least some testing with "baseline only" wasm"
                targets['wasm-32-wasmrt-wasm_mvponly/wabt'].extend(
                    ['internal', 'correctness', 'generator', 'error', 'warning'])

            if builder_type.handles_wasm_v8():
                # TODO: this is a horrid hack. For now, we want to test JIT with both WABT and V8.
                # Add as a horrible wart on the target string.
                targets['wasm-32-wasmrt-wasm_simd128/v8'].extend(
                    ['internal', 'correctness', 'generator', 'error', 'warning'])

            # WABT (and thus WASM JIT) can't handle code build with wasm_threads yet,
            # so only test Generator here
            targets['wasm-32-wasmrt-wasm_simd128-wasm_threads'].extend(
                ['generator', 'apps'])

    if builder_type.handles_webgpu():
        # Most apps can't handle wasm builds yet.
        targets['wasm-32-wasmrt-webgpu'].extend(['generator'])

    return targets


def is_time_critical_test(test):
    # Return true if the test label (or single-test name) is 'time critical' and must
    # be run with an exclusive lock on the buildbot (typically, performance tests)
    return test in ['performance', 'autoschedulers_cpu', 'autoschedulers_gpu', 'autoschedulers_cuda']


def short_target(halide_target):
    s = halide_target.split('-')
    if len(s) == 1:
        return s[0]
    elif len(s) == 2:
        return '%s-%s' % (s[0], s[1])
    elif len(s) == 3:
        return '%s-%s-%s' % (s[0], s[1], s[2])
    elif len(s) > 3:
        return '%s-%s-%s…' % (s[0], s[1], s[2])
    else:
        return '<unknown>'


def add_halide_cmake_test_steps(factory, builder_type):
    parallelism = Property('WORKER_BUILD_PARALLELISM')

    labels = get_test_labels(builder_type)

    source_dir = get_halide_source_path()
    build_dir = get_halide_build_path()
    install_dir = get_halide_install_path(builder_type)  # NOQA

    # Since we need to do at least a partial rebuild for each different target,
    # we want to group things by target. Do host first, followed by a key-sorted
    # order, to ensure predictability.
    keys = list(labels.keys())
    keys.remove('host')
    keys.sort()
    keys.insert(0, 'host')

    # A bit of an ugly hack here -- if we are doing serialization testing
    # via JIT, do it right after `host` so that very little needs rebuilding.
    # To accomplish this we insert a dummy target in the list.
    _FAKE_SERIALIZATION_JIT_TEST_TARGET = 'host-serialization_jit_testing'
    if builder_type.handles_serialization_jit_testing():
        keys.insert(1, _FAKE_SERIALIZATION_JIT_TEST_TARGET)

    for halide_target in keys:
        desc = 'T=%s' % short_target(halide_target)

        if halide_target == _FAKE_SERIALIZATION_JIT_TEST_TARGET:
            halide_target = 'host'
            desc = 'T=host-SERIALIZE_TEST'
            extra_cmake_defs = {'WITH_SERIALIZATION_JIT_ROUNDTRIP_TESTING': 'ON'}
        else:
            extra_cmake_defs = {'WITH_SERIALIZATION_JIT_ROUNDTRIP_TESTING': 'OFF'}

        # HL_TARGET is now ignored by CMake builds, no need to set
        # (must specify -DHalide_TARGET to CMake instead)
        # env['HL_TARGET'] = halide_target
        env = extend_property('env', HL_JIT_TARGET=halide_target)

        # Do this *before* splitting the horrid wasm-specific target string
        test_labels = labels[halide_target]

        # wasm targets must ensure that the EMSDK (emcc, etc) are added to the
        # active env.
        wasm_jit = None
        if halide_target.startswith("wasm-"):
            halide_target, sep, wasm_jit = halide_target.partition('/')
            # Re-set HL_JIT_TARGET with the de-warted target string
            env = extend_property('env', HL_JIT_TARGET=halide_target)
            if wasm_jit:
                desc = '%s + T=%s' % (wasm_jit, short_target(halide_target))

        if not wasm_jit:
            wasm_jit = 'wabt'

        cmake_defs = get_halide_cmake_definitions(builder_type,
                                                  halide_target=halide_target,
                                                  wasm_jit=wasm_jit,
                                                  extra_cmake_defs=extra_cmake_defs)

        factory.addStep(
            CMake(name='Reconfigure for %s' % desc,
                  description='Reconfigure for %s' % desc,
                  locks=[performance_lock.access('counting')],
                  haltOnFailure=True,
                  env=env,
                  workdir=build_dir,
                  path=source_dir,
                  generator=get_cmake_generator(builder_type),
                  definitions=cmake_defs,
                  options=get_halide_cmake_options(builder_type, build_dir)))

        factory.addStep(
            ShellCommand(name='Rebuild for %s' % desc,
                         description='Rebuild Halide for %s' % desc,
                         locks=[performance_lock.access('counting')],
                         haltOnFailure=True,
                         workdir=build_dir,
                         env=env,
                         command=get_cmake_build_command(builder_type, build_dir, targets=['all', 'install'])))

        do_apps = 'apps' in test_labels
        if do_apps:
            test_labels.remove('apps')

        if not builder_type.handles_python():
            if 'python' in test_labels:
                test_labels.remove('python')

            # TODO : some of the apps require python, so we must skip them for now also
            do_apps = False

        parallel_test_labels = [
            test for test in test_labels if not is_time_critical_test(test)]
        exclusive_test_labels = [test for test in test_labels if is_time_critical_test(test)]

        if parallel_test_labels:
            if len(parallel_test_labels) > 2:
                test_set = ','.join([s[0] for s in parallel_test_labels])
            else:
                test_set = ', '.join(parallel_test_labels)

            # Build up some special cases to exclude
            exclude_tests = []
            if builder_type.os == 'windows' or builder_type.os == 'linux':
                # TODO: disable lens_blur on windows for now due to
                #   https://bugs.llvm.org/show_bug.cgi?id=46176
                #   and also due to Windows testbots having inadequate GPU RAM
                #   and also due to Linux testbots having inadequate GPU RAM
                exclude_tests.append('interpolate')
                exclude_tests.append('lens_blur')
                exclude_tests.append('unsharp')

            if builder_type.os == 'linux' or builder_type.bits == 32:
                # TODO: disable tutorial_lesson_12_using_the_gpu (both C++ and python) on linux and 32-bit
                exclude_tests.append('tutorial_lesson_12')

            if builder_type.sanitizer == 'asan':
                # lesson 19 can trigger memory leaks in some of the GPU device drivers,
                # so just exclude it when doing ASAN
                exclude_tests.append('tutorial_lesson_19')

            if builder_type.arch == 'arm' or builder_type.bits == 32:
                # TODO: disable lesson_19 on arm32
                # https://github.com/halide/Halide/issues/5224
                exclude_tests.append('tutorial_lesson_19')

            factory.addStep(
                CTest(name='Test %s %s' % (test_set, desc),
                      description='Test %s %s' % (test_set, desc),
                      locks=[performance_lock.access('counting')],
                      workdir=build_dir,
                      env=env,
                      timeout=3600,
                      labels=parallel_test_labels,
                      exclude_tests=exclude_tests,
                      jobs=parallelism,
                      **get_ctest_options(builder_type, build_dir)))

        if exclusive_test_labels:
            test_set = ','.join([s.replace('autoschedulers_', 'a_') for s in exclusive_test_labels])
            factory.addStep(
                CTest(name='Test %s %s' % (test_set, desc),
                      description='Test %s %s' % (test_set, desc),
                      locks=[performance_lock.access('exclusive')],
                      workdir=build_dir,
                      env=env,
                      timeout=3600,
                      labels=exclusive_test_labels,
                      **get_ctest_options(builder_type, build_dir)))

        if do_apps:
            apps_build_dir = get_halide_build_path("apps")
            apps_source_dir = get_halide_source_path("apps")

            # We currently don't attempt to build any of the apps with wasm
            apps_cmake_defs = get_halide_cmake_definitions(builder_type, halide_target=halide_target)
            apps_cmake_defs['CMAKE_PREFIX_PATH'] = get_halide_install_path(builder_type)

            # apps/hannk is expensive to build, and doesn't (yet) build on all systems, so special-case it here
            want_hannk = (builder_type.has_tflite() and not halide_target.startswith("wasm-"))
            apps_cmake_defs['ENABLE_APPS_HANNK'] = 'ON' if want_hannk else 'OFF'

            factory.addStep(
                CMake(name='Configure apps for %s' % desc,
                      description='Configure apps for %s' % desc,
                      locks=[performance_lock.access('counting')],
                      haltOnFailure=True,
                      env=env,
                      workdir=apps_build_dir,
                      path=apps_source_dir,
                      generator=get_cmake_generator(builder_type),
                      definitions=apps_cmake_defs,
                      options=get_halide_cmake_options(builder_type, build_dir)))

            factory.addStep(
                ShellCommand(name='Build apps for %s' % desc,
                             description='Build apps for %s' % desc,
                             locks=[performance_lock.access('counting')],
                             haltOnFailure=True,
                             workdir=apps_build_dir,
                             env=env,
                             command=get_cmake_build_command(builder_type, apps_build_dir)))

            # Note: do *not* run the apps/ tests in parallel; many of them expect
            # to make full use of all cores, and running in parallel will just slow
            # things down.

            exclude_tests = []
            if builder_type.os == 'windows':
                # TODO: disable lens_blur_filter on windows for now due to
                # https://github.com/halide/Halide/issues/5552
                exclude_tests.append('lens_blur_filter')

            factory.addStep(
                CTest(name='Test apps for %s' % desc,
                      description='Test apps for %s' % desc,
                      locks=[performance_lock.access('exclusive')],
                      workdir=apps_build_dir,
                      env=env,
                      timeout=3600,
                      exclude_tests=exclude_tests,
                      exclude_labels=['slow_tests'],
                      **get_ctest_options(builder_type, apps_build_dir)))


def create_halide_make_factory(builder_type):
    assert builder_type.os != 'windows'

    make_threads = Property('WORKER_BUILD_PARALLELISM')
    build_dir = get_halide_build_path()

    factory = BuildFactory()
    # We never enable sanitizers for Make builds here (only for CMake)
    add_env_setup_step(factory, builder_type, enable_ccache=True)

    # It's never necessary to use get_msvc_config_steps() for Make,
    # since we never use Make with MSVC

    add_get_halide_source_steps(factory, builder_type)

    # Force a full rebuild of Halide every time
    factory.addStep(RemoveDirectory(name="Remove Halide Build Dir",
                                    locks=[performance_lock.access('counting')],
                                    dir=build_dir))

    target_label_pairs = [('host', 'build_tests')]
    for halide_target, labels_for_target in get_test_labels(builder_type).items():

        # For Make we skip every target that isn't plain 'host'
        if halide_target != 'host':
            continue

        _labels_to_skip = [
            # auto_schedule and performance requires exclusive machine access and isn't worth it for Make
            "autoschedulers_cpu",
            "autoschedulers_gpu",
            "autoschedulers_cuda",
            "performance",
            # Make no longer provides support for building the Python bindings,
            # regardless of builder_type.handles_python()
            "python",
        ]

        if builder_type.bits == 32:
            # Don't test autoschedulers on 32-bit systems via Make;
            # it's not set up 100% correctly for crosscompilation there
            # and the CMake-based coverage is fine.
            _labels_to_skip.extend(['autoschedulers_cpu', 'autoschedulers_gpu', 'autoschedulers_cuda'])

        for label in labels_for_target:
            if label in _labels_to_skip:
                continue
            target_label_pairs.append((halide_target, label))

    for halide_target, label in target_label_pairs:
        env = extend_property('env',
                              LLVM_CONFIG=get_llvm_install_path(builder_type, 'bin/llvm-config'),
                              HL_TARGET=halide_target,
                              HL_JIT_TARGET=halide_target)

        if is_time_critical_test(label):
            p = 1
            lock_mode = 'exclusive'
        else:
            p = make_threads
            lock_mode = 'counting'

        if label != 'build_tests':
            label = 'test_%s' % label

        factory.addStep(ShellCommand(name='make ' + label,
                                     description=label + ' ' + halide_target,
                                     locks=[performance_lock.access(lock_mode)],
                                     workdir=build_dir,
                                     env=env,
                                     haltOnFailure=False,
                                     command=['make',
                                              '-f', get_halide_source_path('Makefile'),
                                              '-j', p,
                                              label],
                                     timeout=3600))
    return factory


def create_halide_cmake_factory(builder_type):
    factory = BuildFactory()
    add_env_setup_step(factory, builder_type)
    add_get_halide_source_steps(factory, builder_type)
    add_halide_cmake_build_steps(factory, builder_type)
    add_halide_cmake_test_steps(factory, builder_type)

    # If everything else looks ok, build a distrib.
    if builder_type.purpose == Purpose.halide_nightly:
        add_halide_cmake_package_steps(factory, builder_type)

    return factory


def create_halide_factory(builder_type):
    if builder_type.buildsystem == BuildSystem.cmake:
        return create_halide_cmake_factory(builder_type)
    else:
        return create_halide_make_factory(builder_type)


def get_interesting_halide_targets():
    for arch in ['arm', 'x86']:
        for bits in [32, 64]:
            for os in ['linux', 'osx', 'windows']:
                if arch == 'arm' and os == 'windows':
                    # No buildbots for windows-on-arm (yet)
                    continue
                if os == 'osx' and bits != 64:
                    # osx is 64-bit only, period
                    continue
                yield arch, bits, os


def create_halide_builder(arch, bits, os, halide_branch, llvm_branch, purpose, buildsystem=BuildSystem.cmake):
    # Always do a build with no sanitizers
    sanitizers = [None]

    # Also build with sanitizers (but not if we are doing nightlies)
    if purpose != Purpose.halide_nightly:
        sanitizers.extend(_SANITIZERS)

    for san in sanitizers:
        builder_type = BuilderType(arch, bits, os, halide_branch, llvm_branch, purpose, san, buildsystem)
        if san and purpose == Purpose.llvm_nightly:
            continue
        if san and not builder_type.handles_sanitizers():
            continue

        workers = builder_type.get_worker_names()
        builder = BuilderConfig(name=builder_type.builder_label(),
                                workernames=workers,
                                factory=create_halide_factory(builder_type),
                                collapseRequests=True,
                                # We need counting access to our llvm branch during Halide builds.
                                # (We could probably get by with access during only a subset of
                                # our steps, but there doesn't appear to be a way to group
                                # lock requests across multiple-but-not-all-steps in a Build.)
                                locks=[llvm_build_locks[llvm_branch + str(bits)].access('counting')],
                                tags=builder_type.builder_tags())
        builder.builder_type = builder_type
        yield builder


def create_halide_builders():
    for arch, bits, os in get_interesting_halide_targets():
        # Create builders for build + package of Halide master + release branches
        # (but only against their 'native' LLVM versions)
        for halide_branch in HALIDE_NIGHTLIES:
            for llvm_branch in LLVM_FOR_HALIDE[halide_branch]:
                yield from create_halide_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.halide_nightly)

        # Create the builders for testing pull requests to releases.
        for halide_branch in _HALIDE_RELEASES:
            for llvm_branch in LLVM_FOR_HALIDE[halide_branch]:
                yield from create_halide_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.halide_testbranch)

        # Create the builders for testing pull requests to main.
        yield from create_halide_builder(arch, bits, os, HALIDE_MAIN, LLVM_MAIN, Purpose.halide_testbranch)

    # Also test Makefiles on x86-linux & osx (but only on Halide main) to ensure they
    # stay healthy. (Note: deliberately skip arm-linux, since they are the slowest bots.)
    yield from create_halide_builder('x86', 64, 'linux', HALIDE_MAIN, LLVM_MAIN,
                                     Purpose.halide_testbranch, BuildSystem.make)
    yield from create_halide_builder('x86', 32, 'linux', HALIDE_MAIN, LLVM_MAIN,
                                     Purpose.halide_testbranch, BuildSystem.make)
    yield from create_halide_builder('x86', 64, 'osx', HALIDE_MAIN, LLVM_MAIN,
                                     Purpose.halide_testbranch, BuildSystem.make)
    yield from create_halide_builder('arm', 64, 'osx', HALIDE_MAIN, LLVM_MAIN,
                                     Purpose.halide_testbranch, BuildSystem.make)

    # Test pull requests for Halide master against the current and previous LLVM, for at least one target.
    for llvm_branch in LLVM_BRANCHES:
        if abs(LLVM_BRANCHES[llvm_branch].version.major - LLVM_BRANCHES[LLVM_MAIN].version.major) in [1, 2]:
            yield from create_halide_builder('x86', 64, 'linux', HALIDE_MAIN, llvm_branch, Purpose.halide_testbranch)


def create_halide_scheduler(halide_branch):
    def is_halide_base_branch(br):
        return any(br == hl.ref for hl in HALIDE_BRANCHES.values())

    def is_halide_pr_branch(br):
        # If it's not one of the well-known branches, assume it's a pull request
        return not is_halide_base_branch(br)

    def github_base_branch_matches(change):
        ref = change.properties.getProperty('basename')
        return ref == HALIDE_BRANCHES[halide_branch].ref

    # ----- nightlies
    builders = [b for b in c['builders']
                if b.builder_type.halide_branch == halide_branch and b.builder_type.purpose == Purpose.halide_nightly]
    if builders:
        builder_names = [str(b.name) for b in builders]

        # Start the Halide nightlies at 11PM Pacific; our buildbot master uses UTC for
        # cron, so that's 0600. Note that this is (deliberately) well before
        # the LLVM nightlies get built (currently 1am start); the idea is
        # that Halide nightlies get built using the previous day's LLVM
        # nightlies, on the assumption that those are more likely to get at
        # least some test coverage (via testbranch) to minimize breakage.
        yield Nightly(
            name='halide-package-' + halide_branch,
            codebases=['halide'],
            builderNames=builder_names,
            change_filter=ChangeFilter(codebase='halide'),
            hour=6,
            minute=0)

        yield ForceScheduler(
            name='force-halide-nightly-' + halide_branch,
            builderNames=builder_names,
            codebases=['halide'])

    # ----- testbranch
    builders = [b for b in c['builders']
                if b.builder_type.halide_branch == halide_branch
                and b.builder_type.purpose == Purpose.halide_testbranch]
    if builders:
        # NOT SingleBranchScheduler, because this can process changes from many branches (all PRs)
        builder_names = [str(b.name) for b in builders]
        yield AnyBranchScheduler(
            name='halide-testbranch-' + halide_branch,
            codebases=['halide'],
            change_filter=ChangeFilter(category='pull', codebase='halide',
                                       branch_fn=is_halide_pr_branch,
                                       filter_fn=github_base_branch_matches),
            treeStableTimer=60 * 5,  # seconds
            builderNames=builder_names)

        yield ForceScheduler(
            name='force-halide-testbranch-' + halide_branch,
            builderNames=builder_names,
            codebases=['halide'])


def create_llvm_cmake_factory(builder_type):
    factory = BuildFactory()
    add_env_setup_step(factory, builder_type)
    add_get_llvm_source_steps(factory, builder_type)

    clean_llvm_rebuild = (builder_type.llvm_branch == LLVM_MAIN)

    add_llvm_steps(factory, builder_type, clean_llvm_rebuild)

    return factory


def create_llvm_builders():
    for arch, bits, os in get_interesting_halide_targets():
        # Note that we want these Builders to run on *every* eligible worker;
        # the goal is to ensure that all LLVM builds are updated locally
        # on all of the workers.
        for llvm_branch in LLVM_BRANCHES:
            builder_type = BuilderType(arch, bits, os, None, llvm_branch, Purpose.llvm_nightly)
            for w in builder_type.get_worker_names():
                # Note that we need the builder name to be unique across workers,
                # but we want the builddir on the *worker* side to be the same for all workers
                # (to simplify things).
                label = builder_type.llvm_builder_label()
                builder = BuilderConfig(name="%s/%s" % (label, w),
                                        workerbuilddir=label,
                                        workernames=[w],
                                        factory=create_llvm_cmake_factory(builder_type),
                                        collapseRequests=True,
                                        # We want exclusive access to this workerlock
                                        # thru all this Builder's steps. (We could probably
                                        # get by with holding it just during the install phase,
                                        # but we'd have to finesse some details like removing
                                        # the old install directory within the lock, and this
                                        # is much simpler.)
                                        locks=[llvm_build_locks[llvm_branch + str(bits)].access('exclusive')],
                                        tags=builder_type.builder_tags())
                builder.builder_type = builder_type
                yield builder


def create_llvm_scheduler(llvm_branch):
    builders = [str(b.name) for b in c['builders']
                if b.builder_type.llvm_branch == llvm_branch and b.builder_type.purpose == Purpose.llvm_nightly]
    # Start every day at 1AM Pacific; our buildbot use UTC for cron, so that's 0800
    yield Nightly(
        name=f'llvm-nightly-{LLVM_BRANCHES[llvm_branch].version.major}',
        codebases=['llvm'],
        builderNames=builders,
        change_filter=ChangeFilter(codebase='llvm'),
        hour=8,
        minute=0)

    for b in builders:
        yield ForceScheduler(
            name='force-llvm-nightly-' + b.replace('/', '_'),
            codebases=['llvm'],
            builderNames=[b])


def create_builders():
    yield from create_llvm_builders()
    yield from create_halide_builders()


def create_schedulers():
    for llvm_branch in LLVM_BRANCHES:
        yield from create_llvm_scheduler(llvm_branch)

    for halide_branch in HALIDE_BRANCHES:
        yield from create_halide_scheduler(halide_branch)


c['builders'] = list(create_builders())
c['schedulers'] = list(create_schedulers())


# Set the builder priorities


def prioritize_builders(buildmaster, builders):
    def importance(builder):
        builder_type = builder.config.builder_type
        assert builder_type

        # LLVM nightlies run only once a day (late at night) and should always
        # get priority over everything else.
        if builder_type.purpose == Purpose.llvm_nightly:
            return 0

        # Branch testers all need to come back before we can merge a PR,
        # so they all have equal next-highest priority.
        if builder_type.purpose == Purpose.halide_testbranch:
            return 1

        # non-branch testers are mostly used for bisecting failures that
        # didn't show up in the branch testers and doing binary
        # releases. We care most about the most recently-released llvm so
        # that we have a full set of builds for releases, then llvm main
        # for bisection, then older llvm versions.
        if builder_type.llvm_branch in LLVM_FOR_HALIDE[HALIDE_RELEASE_16]:
            return 2
        if builder_type.llvm_branch in LLVM_FOR_HALIDE[HALIDE_RELEASE_17]:
            return 3
        if builder_type.llvm_branch in LLVM_FOR_HALIDE[HALIDE_MAIN]:
            return 4
        return 5

    return list(sorted(builders, key=importance))


c['prioritizeBuilders'] = prioritize_builders


# GitHub pull request filter

class SafeGitHubEventHandler(GitHubEventHandler):
    def handle_push(self, payload, event):
        ref = payload['ref']
        if re.match(r"^refs/(heads|tags)/(master|main|release/\d+\.x)$", ref):
            return super().handle_push(payload, event)
        else:
            print(f'SafeGitHubEventHandler: ignoring push event for ref: {ref}')
            return self.skip()

    def handle_pull_request(self, payload, event):
        pr = payload['pull_request']
        try:
            # Skip anything with the 'skip_buildbots' label
            if any(label['name'] == 'skip_buildbots' for label in pr['labels']):
                # print("PR %s was skipped due to skip_buildbots" % str(pr['html_url']))
                return self.skip()

            # Test anything (even external) that has 'halidebuildbots' as a reviewer.
            if any(r['login'] == 'halidebuildbots' for r in pr['requested_reviewers']):
                # print("PR %s was handled due halidebuildbots" % str(pr['html_url']))
                if payload['action'] == 'review_requested':
                    # Pretend it's a synchronize event instead since private buildbot code
                    # rejects review_requested for no apparent reason.
                    payload['action'] = 'synchronize'
                return super().handle_pull_request(payload, event)

            # Skip external pull requests that originate from untrusted forks
            trusted_repos = (
                'halide/Halide',  # the primary repository is obviously trusted
                'CodeLinaro/Halide',  # the Qualcomm open-source staging fork is trustworthy
            )
            if pr['head']['repo']['full_name'] not in trusted_repos:
                # print("PR %s was skipped due to being external:" % str(pr['head']['repo']['full_name']))
                return self.skip()

            # print("PR %s is being handled normally" % str(pr['html_url']))
            return super().handle_pull_request(payload, event)

        except KeyError as e:
            print(f'SafeGitHubEventHandler: malformed payload: {payload}')
            print(f'SafeGitHubEventHandler: missing key "{e}"')
            return self.skip()

    @staticmethod
    def skip():
        return [], 'git'


# WEB SERVER

# 'protocols' contains information about protocols which master will use for communicating with workers.
# You must define at least 'port' option that workers could connect to your master with this protocol.
# 'port' must match the value configured into the buildworkers (with their --master option)
c['protocols'] = {'pb': {'port': 9990}}

authz = Authz(
    allowRules=[ems.ForceBuildEndpointMatcher(role="admins"),
                ems.StopBuildEndpointMatcher(role="admins"),
                ems.RebuildBuildEndpointMatcher(role="admins"),
                ems.EnableSchedulerEndpointMatcher(role="admins")],
    roleMatchers=[RolesFromUsername(roles=["admins"], usernames=["halidenightly"])])

c['www'] = dict(
    auth=UserPasswordAuth({'halidenightly': WWW_PASSWORD}),
    authz=authz,
    port=8012,
    change_hook_dialects={
        'github': {
            'secret': WEBHOOK_SECRET,
            'codebase': 'halide',
            'skips': [],
            'class': SafeGitHubEventHandler,
            # 'github_property_whitelist': ['github.base.ref'],
        },
    },
)

# PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = 'Halide'
c['titleURL'] = 'http://halide-lang.org'

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = 'https://buildbot.halide-lang.org/master/'

# DB URL

c['db'] = {
    # This specifies what database buildbot uses to store its state.  You can leave
    # this at its default for all but the largest installations.
    'db_url': 'sqlite:///state.sqlite',
}

# GitHub Integration

# Only testbranch builders need to be considered here
builders = [str(b.name) for b in c['builders'] if b.builder_type.purpose != Purpose.llvm_nightly]
generator = BuildStartEndStatusGenerator(builders=builders,
                                         start_formatter=MessageFormatterRenderable('Build started.'),
                                         end_formatter=MessageFormatterRenderable('Build done.'))

gs = GitHubStatusPush(token=GITHUB_TOKEN,
                      context=Interpolate("buildbot/%(prop:buildername)s"),
                      generators=[generator],
                      verbose=True)
c['services'] = [gs]

# Disable sending usage data
c['buildbotNetUsageData'] = None