Show all stable builders on the release status page; add Disconnected stable workers (#525)

Author: encukou

master/custom/release_dashboard.py

@@ -22,7 +22,8 @@ def get_release_status_app(buildernames):
     def get_release_status():
         builders = release_status_app.buildbot_api.dataGet("/builders")
 
-        failed_builds_by_branch = {}
+        failed_builds_by_branch_and_tier = {}
+        disconnected_workers = {}
 
         for builder in builders:
             if builder["name"] not in buildernames:
@@ -31,15 +32,24 @@ def get_release_status():
             if "stable" not in builder["tags"]:
                 continue
 
-            branch = [tag for tag in builder["tags"] if "3." in tag]
+            for worker in release_status_app.buildbot_api.dataGet(
+                ("builders", builder["builderid"], "workers"),
+            ):
+                if not worker["connected_to"]:
+                    disconnected_workers[worker["name"]] = worker
+
+            branch = None
+            tier = 'no tier'
+            for tag in builder["tags"]:
+                if "3." in tag:
+                    branch = tag
+                if tag.startswith('tier-'):
+                    tier = tag
 
             if not branch:
                 continue
 
-            (branch,) = branch
-
-            if branch not in failed_builds_by_branch:
-                failed_builds_by_branch[branch] = []
+            failed_builds_by_tier = failed_builds_by_branch_and_tier.setdefault(branch, {})
 
             endpoint = ("builders", builder["builderid"], "builds")
             last_build = release_status_app.buildbot_api.dataGet(
@@ -56,14 +66,29 @@ def get_release_status():
             if last_build["results"] != FAILED_BUILD_STATUS:
                 continue
 
-            failed_builds_by_branch[branch].append((builder, last_build))
+            failed_builds = failed_builds_by_tier.setdefault(tier, [])
+            failed_builds.append((builder, last_build))
+
+        def tier_sort_key(item):
+            tier, data = item
+            if tier == 'no tier':
+                return 'zzz'  # sort last
+            return tier
+        failed_builders = []
+        for branch, failed_builds_by_tier in failed_builds_by_branch_and_tier.items():
+            failed_builders.append((
+                branch,
+                sorted(failed_builds_by_tier.items(), key=tier_sort_key)
+            ))
+        failed_builders.sort(reverse=True)
 
         generated_at = datetime.datetime.now(tz=datetime.timezone.utc)
-        failed_builders = sorted(failed_builds_by_branch.items(), reverse=True)
+
         return render_template(
             "releasedashboard.html",
             failed_builders=failed_builders,
             generated_at=generated_at,
+            disconnected_workers=sorted(disconnected_workers.items()),
         )
 
     @release_status_app.route("/index.html")

master/custom/templates/releasedashboard.html

@@ -5,12 +5,14 @@
         <h1>Branch release status dashboard</h1>
     </div>
 
+    <h2>Failing stable builders</h2>
+
     <div class="container">
         <div class="col-sm-12">
             <div class="row">
-                {% for branch, info in failed_builders %}
+                {% for branch, info_by_tier in failed_builders %}
                     <div class="col-md-4 ng-scope">
-                        {% if not info %}
+                        {% if not info_by_tier %}
                             <div class="panel panel-success">
                                 <div class="panel-heading">
                                     <div style="text-align: center;">
@@ -30,19 +32,22 @@ <h4 class="panel-title">
                                 <div class="panel-heading">
                                     <div style="text-align: center;">
                                         <h4 class="panel-title">
-                                            Branch {{ branch }} status: Not Releaseable
+                                            Branch {{ branch }} status:
                                         </h4>
                                     </div>
                                 </div>
                                 <div class="panel-body">
-                                    {% for builder, build in info %}
-                              <!-- BROKEN?      <div> <buildsummary buildid="{{build.buildid}}" condensed="1"/> </div> -->
-                                        <div>
-                                            <a class="badge-status badge results_{{build.results_text | upper}}"
-                                               href="#/builders/{{build.builderid}}/builds/{{build.number}}">
-                                               {{build.number}}
-                                            </a>
-                                        </div>
+                                    {% for tier, info in info_by_tier %}
+                                        <h5>{{ tier }}</h5>
+                                        {% for builder, build in info %}
+                                <!-- BROKEN?      <div> <buildsummary buildid="{{build.buildid}}" condensed="1"/> </div> -->
+                                            <div>
+                                                <a class="badge-status badge results_{{build.results_text | upper}}"
+                                                href="#/builders/{{build.builderid}}/builds/{{build.number}}">
+                                                {{ builder.name }}: #{{ build.number }}
+                                                </a>
+                                            </div>
+                                        {% endfor %}
                                     {% endfor %}
                                 </div>
                             </div>
@@ -52,6 +57,22 @@ <h4 class="panel-title">
             </div>
         </div>
     </div>
+
+    <h2>Disconnected stable workers</h2>
+
+    <div class="container">
+        {% for name, worker in disconnected_workers %}
+            <div class="row">
+                -
+                <a href="#/workers/{{ worker.workerid }}">
+                    {{ name }}
+                </a>
+            </div>
+        {% else %}
+            None.
+        {% endfor %}
+    </div>
+
     <div class="container">
         <small>Generated at <time id="generatedAt" datetime="{{generated_at}}">{{generated_at}}</time></small>
     </div>

master/master.cfg

@@ -137,16 +137,6 @@ GIT_KWDS = {
 c["builders"] = []
 c["schedulers"] = []
 
-# Match builder name (including the branch name) of builders that should never
-# emit any notification. Basically, notifications are disabled on workers
-# which were never green (tests always failed).
-NO_NOTIFICATION = (
-    "Alpine Linux",
-    "Cygwin",
-    # UBSan always failed on 3.6, 3.7 and 3.x
-    "AMD64 Clang UBSan 3.",
-)
-
 parallel = {w.name: f"-j{w.parallel_tests}" for w in WORKERS if w.parallel_tests}
 extra_factory_args = {
     "ware-gentoo-x86": {
@@ -248,14 +238,13 @@ for branch_num, (git_url, branchname, git_branch) in enumerate(git_branches):
             refleakbuildernames.append(buildername)
         else:
             buildernames.append(buildername)
-        if all(pattern not in buildername for pattern in NO_NOTIFICATION):
+        # disable notifications for unstable builders
+        # (all these lists are the same now, but we might need to
+        # diverge gain later.)
+        if stability == STABLE:
             mail_status_builders.append(buildername)
-            # disable GitHub notifications for unstable builders
-            if stability == STABLE:
-                github_status_builders.append(buildername)
-                # Only Tier-1 and Tier-2 builders can block a release
-                if tier in (TIER_1, TIER_2):
-                    release_status_builders.append(buildername)
+            github_status_builders.append(buildername)
+            release_status_builders.append(buildername)
         c["builders"].append(
             util.BuilderConfig(
                 name=buildername,