Remove nightly Halide builds from master.cfg

Author: alexreinking

master/master.cfg

@@ -8,7 +8,6 @@ import os
 import re
 from collections import defaultdict
 from dataclasses import dataclass
-from enum import Enum
 from functools import reduce
 from pathlib import Path
 
@@ -23,7 +22,6 @@ from buildbot.reporters.github import GitHubStatusPush
 from buildbot.reporters.message import MessageFormatterRenderable
 from buildbot.schedulers.basic import AnyBranchScheduler
 from buildbot.schedulers.forcesched import ForceScheduler
-from buildbot.schedulers.timed import Nightly
 from buildbot.steps.cmake import CMake
 from buildbot.steps.master import SetProperties
 from buildbot.steps.shell import SetPropertyFromCommand, ShellCommand
@@ -135,15 +133,6 @@ HALIDE_BRANCHES = {
     HALIDE_RELEASE_21: VersionedBranch(ref="release/21.x", version=Version(21, 0, 0)),
 }
 
-# This lists the Halide branch(es) for which we want to build nightlies;
-# it's usually desirable to constrain these to save buildbot time (esp on the slower bots)
-# and avoid branches that aren't changing much (i.e. -- recent releases that aren't
-# likely to need new updates soon).
-HALIDE_NIGHTLIES = [
-    HALIDE_MAIN,
-    HALIDE_RELEASE_21,
-]
-
 # Given a halide branch, return the 'native' llvm version we expect to use with it.
 # For halide release branches, this is the corresponding llvm release branch; for
 # halide main, it's llvm main.
@@ -200,11 +189,6 @@ performance_lock = WorkerLock("performance_lock", maxCount=9999)
 # only take place on one worker.
 
 
-class Purpose(Enum):
-    nightly = 1
-    testbranch = 2
-
-
 class BuilderType:
     """
     A class to encapsulate the settings for a specific Builder.
@@ -213,7 +197,6 @@ class BuilderType:
     It includes:
     - Halide 'target' in the form of arch-bits-os
     - LLVM branch to be used
-    - halide-nightly vs halide-testbranch
     - sanitizers vs none
 
     It doesn't currently include any 'features' because we don't currently
@@ -224,7 +207,7 @@ class BuilderType:
     setup. (If we ever need to do so, compiler should be added to this.)
     """
 
-    def __init__(self, arch, bits, os, halide_branch, llvm_branch, purpose, sanitizer=None):
+    def __init__(self, arch, bits, os, halide_branch, llvm_branch, sanitizer=None):
         assert arch in ["arm", "x86"]
         assert bits in [32, 64]
         assert os in ["linux", "windows", "osx"]
@@ -235,15 +218,10 @@ class BuilderType:
         self.os = os
         self.halide_branch = halide_branch
         self.llvm_branch = llvm_branch
-        self.purpose = purpose
         self.sanitizer = sanitizer
 
         assert self.halide_branch, "halide_branch is required"
         assert self.halide_branch in HALIDE_BRANCHES, f"unknown branch {self.halide_branch}"
-        assert (
-            self.purpose == Purpose.testbranch  # if not testbranch...
-            or self.llvm_branch in LLVM_FOR_HALIDE[self.halide_branch]
-        )
 
         if self.sanitizer:
             assert self.sanitizer in _SANITIZERS
@@ -329,10 +307,6 @@ class BuilderType:
         a = ["halide"]
         if self.sanitizer:
             a.append(self.sanitizer)
-        if self.purpose == Purpose.testbranch:
-            a.append("testbranch")
-        elif self.purpose == Purpose.nightly:
-            a.append("nightly")
         a.append(self.halide_branch)
         if self.halide_branch == HALIDE_MAIN:
             # Halide master is built against multiple LLVM versions,
@@ -871,86 +845,6 @@ def add_build_steps(factory, builder_type):
     )
 
 
-def add_package_steps(factory, builder_type):
-    source_dir = get_source_path()
-
-    target = builder_type.halide_target()
-    ext = "zip" if builder_type.os == "windows" else "tar.gz"
-
-    factory.addStep(
-        SetPropertiesFromCMakeCache(
-            name="Get Halide package version", workdir=get_build_path(), props=["CMAKE_PROJECT_VERSION"]
-        )
-    )
-
-    extra_env = dict(
-        Halide_LLVM_ROOT=Property("HALIDE_LLVM_ROOT"),
-        Halide_VERSION=Property("CMAKE_PROJECT_VERSION"),
-    )
-
-    if builder_type.os == "windows":
-        # TODO: on Windows, we can't use Ninja for packaging (as we do everywhere
-        # else in this cfg) due to a bug in CMake 3.18, so we must use MSBuild;
-        # that means we must use a different build directory entirely. To simplify the
-        # world, we make this a subdir of the real build dir (so it gets cleaned properly).
-        # https://github.com/halide/Halide/issues/5264
-        build_dir = get_build_path("packaging_dir")
-
-        if builder_type.arch == "arm":
-            arch = "ARM" if builder_type.bits == 32 else "ARM64"
-        else:
-            arch = "Win32" if builder_type.bits == 32 else "x64"
-        cmd = [get_source_path("packaging/zip/package.bat"), source_dir, build_dir, arch]
-    else:
-        build_dir = get_build_path()
-        cmd = [get_source_path("packaging/tgz/package.sh"), source_dir, build_dir]
-        if builder_type.arch == "arm" and builder_type.bits == 32 and builder_type.os == "linux":
-            extra_env["CMAKE_TOOLCHAIN_FILE"] = get_source_path("cmake", "toolchain.linux-arm32.cmake")
-
-    factory.addStep(
-        ShellCommand(
-            name="Package Halide",
-            description="Package Halide",
-            workdir=build_dir,
-            env=Merge(Property("env"), extra_env),
-            locks=[performance_lock.access("counting")],
-            haltOnFailure=True,
-            command=cmd,
-        )
-    )
-
-    factory.addStep(
-        FileUpload(
-            name="Upload Halide package",
-            workersrc=Interpolate(f"Halide-%(prop:CMAKE_PROJECT_VERSION)s-{target}.{ext}"),
-            locks=[performance_lock.access("counting")],
-            haltOnFailure=True,
-            workdir=build_dir,
-            mode=0o644,
-            masterdest=Interpolate(
-                f"{ARTIFACTS_DIR}/Halide-%s-{target}-%s.{ext}",
-                Property("CMAKE_PROJECT_VERSION"),
-                Lookup(Property("got_revision"), "halide"),
-            ),
-        )
-    )
-
-    def pkg_version_and_target(path: Path):
-        # Archives names are formatted like: Halide-[version]-[arch]-[commit].[ext]
-        # This grabs "Halide-[version]-[arch]".
-        match = re.match(r"^(.*)-[a-f0-9]+\.(tar\.gz|tgz|zip)", path.name)
-        return match.group(1) if match else None
-
-    factory.addStep(
-        CleanOldFiles(
-            name="Clean old Halide releases",
-            workdir=ARTIFACTS_DIR,
-            locks=[performance_lock.access("counting")],
-            groupfn=pkg_version_and_target,
-        )
-    )
-
-
 # Figure out which "non-cpu" (GPU, DSP, etc) targets this builder can handle.
 # Return (target, is_simulator)
 def get_gpu_dsp_targets(builder_type):
@@ -1341,11 +1235,6 @@ def create_build_factory(builder_type):
     add_install_llvm_step(factory, builder_type)
     add_build_steps(factory, builder_type)
     add_test_steps(factory, builder_type)
-
-    # If everything else looks ok, build a distrib.
-    if builder_type.purpose == Purpose.nightly:
-        add_package_steps(factory, builder_type)
-
     return factory
 
 
@@ -1362,16 +1251,16 @@ def get_interesting_targets():
                 yield arch, bits, os
 
 
-def create_builder(arch, bits, os, halide_branch, llvm_branch, purpose):
+def create_builder(arch, bits, os, halide_branch, llvm_branch):
     # Always do a build with no sanitizers
     sanitizers = [None]
 
-    # Also build with sanitizers (but not if we are doing nightlies)
-    # if purpose != Purpose.nightly:
-    #     sanitizers.extend(_SANITIZERS)
+    # Also build with sanitizers
+    # TODO: disabled for now while we work out packaging issues with the sanitizer runtimes
+    # sanitizers.extend(_SANITIZERS)
 
     for san in sanitizers:
-        builder_type = BuilderType(arch, bits, os, halide_branch, llvm_branch, purpose, san)
+        builder_type = BuilderType(arch, bits, os, halide_branch, llvm_branch, san)
         if san and not builder_type.handles_sanitizers():
             continue
 
@@ -1389,24 +1278,18 @@ def create_builder(arch, bits, os, halide_branch, llvm_branch, purpose):
 
 def create_builders():
     for arch, bits, os in get_interesting_targets():
-        # Create builders for build + package of Halide master + release branches
-        # (but only against their 'native' LLVM versions)
-        for halide_branch in HALIDE_NIGHTLIES:
-            for llvm_branch in LLVM_FOR_HALIDE[halide_branch]:
-                yield from create_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.nightly)
-
         # Create the builders for testing pull requests to releases.
         for halide_branch in _HALIDE_RELEASES:
             for llvm_branch in LLVM_FOR_HALIDE[halide_branch]:
-                yield from create_builder(arch, bits, os, halide_branch, llvm_branch, Purpose.testbranch)
+                yield from create_builder(arch, bits, os, halide_branch, llvm_branch)
 
         # Create the builders for testing pull requests to main.
-        yield from create_builder(arch, bits, os, HALIDE_MAIN, LLVM_MAIN, Purpose.testbranch)
+        yield from create_builder(arch, bits, os, HALIDE_MAIN, LLVM_MAIN)
 
     # Test pull requests for Halide master against the current and previous LLVM, for at least one target.
     for llvm_branch in LLVM_BRANCHES:
         if llvm_branch != LLVM_MAIN:
-            yield from create_builder("x86", 64, "linux", HALIDE_MAIN, llvm_branch, Purpose.testbranch)
+            yield from create_builder("x86", 64, "linux", HALIDE_MAIN, llvm_branch)
 
 
 def create_scheduler(halide_branch):
@@ -1421,47 +1304,18 @@ def create_scheduler(halide_branch):
         ref = change.properties.getProperty("basename")
         return ref == HALIDE_BRANCHES[halide_branch].ref
 
-    # ----- nightlies
-    builders = [
-        b
-        for b in c["builders"]
-        if b.builder_type.halide_branch == halide_branch and b.builder_type.purpose == Purpose.nightly
-    ]
-    if builders:
-        builder_names = [str(b.name) for b in builders]
-
-        # Start the Halide nightlies at 11PM Pacific; our buildbot master uses UTC for
-        # cron, so that's 0600. Note that this is (deliberately) well before
-        # the LLVM nightlies get built (currently 1am start); the idea is
-        # that Halide nightlies get built using the previous day's LLVM
-        # nightlies, on the assumption that those are more likely to get at
-        # least some test coverage (via testbranch) to minimize breakage.
-        yield Nightly(
-            name="halide-package-" + halide_branch,
-            builderNames=builder_names,
-            hour=6,
-            minute=0,
-        )
-
-        yield ForceScheduler(name="force-halide-nightly-" + halide_branch, builderNames=builder_names)
-
-    # ----- testbranch
-    builders = [
-        b
-        for b in c["builders"]
-        if b.builder_type.halide_branch == halide_branch and b.builder_type.purpose == Purpose.testbranch
-    ]
+    builders = [b for b in c["builders"] if b.builder_type.halide_branch == halide_branch]
     if builders:
         # NOT SingleBranchScheduler, because this can process changes from many branches (all PRs)
         builder_names = [str(b.name) for b in builders]
         yield AnyBranchScheduler(
-            name="halide-testbranch-" + halide_branch,
+            name="halide-" + halide_branch,
             change_filter=ChangeFilter(category="pull", branch_fn=is_pr_branch, filter_fn=github_base_branch_matches),
             treeStableTimer=60 * 5,  # seconds
             builderNames=builder_names,
         )
 
-        yield ForceScheduler(name="force-halide-testbranch-" + halide_branch, builderNames=builder_names)
+        yield ForceScheduler(name="force-halide-" + halide_branch, builderNames=builder_names)
 
 
 def create_schedulers():