Report nix-builds using BuildRequests for early reports

Signed-off-by: magic_rb <[email protected]>

Author: MagicRB

buildbot_nix/__init__.py

@@ -364,28 +364,18 @@ def produceEventForBuildRequestsById(
             event: str,
             result: None | int,
     ) -> Generator[Any, object, None]:
-        buildrequest_ids = list(buildrequest_ids)
-
-        def gotBuild(key: int, build: Any) -> None:
-            if build['buildrequestid'] in buildrequest_ids:
-              log.info("got build {key} : {build}", key = key, build = build)
-              # we only care about the first started build
-              buildrequest_ids.remove(int(build['buildrequestid']))
-              if not buildrequest_ids:
-                  del self.consumers[build['buildrequestid']]
-              self.produceEventForBuild(event, build, result)
-
         for buildrequest_id in buildrequest_ids:
-          builds: Any = yield self.master.data.get(('buildrequests', str(buildrequest_id), 'builds'))
-
-          if not builds:
-              # only subscribe to new builds, if there are no existing builds
-              self.consumers[buildrequest_id] = yield self.master.mq.startConsuming(
-                  gotBuild, ('builds', None, None)
-              )
-          # send starts for any exsting builds
-          for build in builds:
-              self.produceEventForBuild(event, build, result)
+            builds: Any = yield self.master.data.get(('buildrequests', str(buildrequest_id), 'builds'))
+
+            # only report with `buildrequets` if there are no builds to report for
+            if not builds:
+              buildrequest: Any = yield self.master.data.get(('buildrequests', str(buildrequest_id)))
+              if result is not None:
+                  buildrequest["results"] = result
+              self.master.mq.produce(("buildrequests", str(buildrequest['buildrequestid']), event), copy.deepcopy(buildrequest))
+            else:
+                for build in builds:
+                    self.produceEventForBuild(event, build, result)
 
     @defer.inlineCallbacks
     def produceEventForBuildById(
@@ -415,7 +405,8 @@ def run(self) -> Generator[Any, Any, None]:
         who's dependencies have completed successfully. If a job fails, we recursively fail every jobs which
         depends on it.
         We also run fake builds for failed evaluations so that they nicely show up in the UI and also Forge
-        reports.
+        reports. The reporting is based on custom events and logic, see `BuildNixEvalStatusGenerator` for
+        the receiving side.
         """
         if self.combine_builds:
             self.produceEventForBuildById("started-nix-build", self.build.buildid, None)
@@ -535,9 +526,6 @@ def run(self) -> Generator[Any, Any, None]:
             scheduler_log.addStdout(
                 f"Found finished build {job.attr}, result {util.Results[result].upper()}\n"
             )
-            log.info(
-                f"Found finished build {job.attr}, result {util.Results[result].upper()}, brids: {brids}"
-            )
             if not self.combine_builds:
                 self.produceEventForBuildRequestsById(brids.values(), "finished-nix-build", result)
 

buildbot_nix/nix_status_generator.py

@@ -9,6 +9,13 @@
 from buildbot.process.build import Build
 from buildbot.reporters.utils import getDetailsForBuild
 from datetime import datetime, timezone
+from buildbot.reporters import utils
+from buildbot.process.buildrequest import BuildRequest
+from buildbot.process.properties import Properties
+from buildbot.process.results import CANCELLED
+from twisted.logger import Logger
+
+log = Logger()
 
 @implementer(IReportGenerator)
 class BuildNixEvalStatusGenerator(BuildStatusGeneratorMixin):
@@ -17,6 +24,8 @@ class BuildNixEvalStatusGenerator(BuildStatusGeneratorMixin):
         ('builds', None, 'finished-nix-eval'),
         ('builds', None, 'started-nix-build'),
         ('builds', None, 'finished-nix-build'),
+        ('buildrequests', None, 'started-nix-build'),
+        ('buildrequests', None, 'canceled-nix-build'),
     ]
 
     compare_attrs = ['start_formatter', 'end_formatter']
@@ -43,34 +52,88 @@ def __init__(
         if self.end_formatter is None:
             self.end_formatter = MessageFormatterRenderable('Build done.')
 
+    # TODO: copy pasted from buildbot, make it static upstream and reuse
+    @staticmethod
     @defer.inlineCallbacks
-    def generate(self, master: BuildMaster, reporter: ReporterBase, key: tuple[str, None | Any, str], build: Build) -> defer.Generator[Any, object, Any]:
-        _, _, event = key
-        is_new = event == 'new'
-
-        formatter = self.start_formatter if is_new else self.end_formatter
-
-        yield getDetailsForBuild(
-            master,
-            build,
-            want_properties=formatter.want_properties,
-            want_steps=formatter.want_steps,
-            want_logs=formatter.want_logs,
-            want_logs_content=formatter.want_logs_content,
+    def partial_build_dict(master: BuildMaster, buildrequest: BuildRequest) -> defer.Generator[Any, object, dict[str, Any]]:
+        brdict: Any = yield master.db.buildrequests.getBuildRequest(buildrequest['buildrequestid'])
+        bdict = {}
+
+        props = Properties()
+        buildrequest = yield BuildRequest.fromBrdict(master, brdict)
+        builder = yield master.botmaster.getBuilderById(brdict["builderid"])
+
+        yield Build.setup_properties_known_before_build_starts(props, [buildrequest], builder)
+        Build.setupBuildProperties(props, [buildrequest])
+
+        bdict['properties'] = props.asDict()
+        yield utils.get_details_for_buildrequest(master, brdict, bdict)
+        return bdict
+
+    # TODO: copy pasted from buildbot, somehow reuse
+    @defer.inlineCallbacks
+    def buildrequest_message(self, master: BuildMaster, build: dict[str, Any]) -> defer.Generator[Any, Any, dict[str, Any]]:
+        patches = self._get_patches_for_build(build)
+        users: list[str] = []
+        buildmsg = yield self.start_formatter.format_message_for_build(
+            master, build, is_buildset=True, mode=self.mode, users=users
         )
 
-        if not self.is_message_needed_by_props(build):
-            return None
+        return {
+            'body': buildmsg['body'],
+            'subject': buildmsg['subject'],
+            'type': buildmsg['type'],
+            'results': build['results'],
+            'builds': [build],
+            "buildset": build["buildset"],
+            'users': list(users),
+            'patches': patches,
+            'logs': [],
+            "extra_info": buildmsg["extra_info"],
+        }
+
+    @defer.inlineCallbacks
+    def generate(self, master: BuildMaster, reporter: ReporterBase, key: tuple[str, None | Any, str], data: Build | BuildRequest) -> defer.Generator[Any, Any, Any]:
+        what, _, event = key
+        log.info("{what} produced {event}", what = what, event = event)
+        if what == "builds":
+          is_new = event == 'new'
+
+          formatter = self.start_formatter if is_new else self.end_formatter
+
+          yield getDetailsForBuild(
+              master,
+              data,
+              want_properties=formatter.want_properties,
+              want_steps=formatter.want_steps,
+              want_logs=formatter.want_logs,
+              want_logs_content=formatter.want_logs_content,
+          )
+
+          if not self.is_message_needed_by_props(data):
+              return None
+
+          report = yield self.build_message(formatter, master, reporter, data)
+          reportT: dict[str, Any] = cast(dict[str, Any], report)
+
+          if event == "started-nix-eval" or event == "finished-nix-eval":
+              reportT["builds"][0]["properties"]["status_name"] = ("nix-eval", "generator")
+          if (event == "started-nix-build" or event == "finished-nix-build") and reportT["builds"][0]["properties"]["status_name"][0] == "nix-eval":
+              reportT["builds"][0]["properties"]["status_name"] = ("nix-build", "generator")
+          if event == "finished-nix-eval" or event == "finished-nix-build":
+              reportT["builds"][0]["complete"] = True
+              reportT["builds"][0]["complete_at"] = datetime.now(tz=timezone.utc)
+
+          return reportT
+        elif what == "buildrequests":
+          build: dict[str, Any] = yield self.partial_build_dict(master, data)
 
-        report = yield self.build_message(formatter, master, reporter, build)
-        reportT: dict[str, Any] = cast(dict[str, Any], report)
+          if event == 'canceled-nix-build':
+            build['complete'] = True
+            build['results'] = CANCELLED
 
-        if event == "started-nix-eval" or event == "finished-nix-eval":
-            reportT["builds"][0]["properties"]["status_name"] = ("nix-eval", "generator")
-        if (event == "started-nix-build" or event == "finished-nix-build") and reportT["builds"][0]["properties"]["status_name"][0] == "nix-eval":
-            reportT["builds"][0]["properties"]["status_name"] = ("nix-build", "generator")
-        if event == "finished-nix-eval" or event == "finished-nix-build":
-            reportT["builds"][0]["complete"] = True
-            reportT["builds"][0]["complete_at"] = datetime.now(tz=timezone.utc)
+          if not self.is_message_needed_by_props(build):
+              return None
 
-        return reportT
+          report = yield self.buildrequest_message(master, build)
+          return report