[bots] Checking for approved workflow runs: handle runs being startup failure (#6329)

If all the runs are startup failure, the function would return true.
However, this can hide the need for those runs to be approved. This
might result in runs being rejected when they shouldn't, but I doubt an
actual PR is going to mangle workflows and still want CI

Err on the side of caution by refusing if any job has a startup failure,
even if there are approved runs

Example: pytorch/pytorch#147774

(I am hash update bot, its a poorly named account since I was originally
going to use for something else and was the fastest account I could get
right now to test this)

There is a startup_failure conclusion, but depending on how you mess up
the workflow syntax, you can either get failure or startup_failure
(https://github.com/pytorch/pytorch/pull/148016/files), and it seems
like if it is going to be a startup_failure if it runs, the workflow
doesn't show up at all when you query about the commit

Author: clee2000

torchci/lib/bot/utils.ts

@@ -270,7 +270,14 @@ export async function hasApprovedPullRuns(
   if (pr_runs == null || pr_runs?.length == 0) {
     return false;
   }
-  return pr_runs.every((run) => run.conclusion != "action_required");
+  return !pr_runs.some(
+    (run) =>
+      run.conclusion === "action_required" ||
+      // See https://github.com/pytorch/test-infra/pull/6329 about difference
+      // between these two
+      run.conclusion === "startup_failure" ||
+      (run.conclusion === "failure" && run.created_at == run.updated_at)
+  );
 }
 
 export async function isFirstTimeContributor(

torchci/test/utils.test.ts

@@ -0,0 +1,93 @@
+import { hasApprovedPullRuns } from "lib/bot/utils";
+import nock from "nock";
+import { Probot } from "probot";
+import triggerInductorTestsBot from "../lib/bot/triggerInductorTestsBot";
+import * as utils from "./utils";
+
+nock.disableNetConnect();
+
+describe("utils: hasApprovedPullRuns", () => {
+  let probot: Probot;
+  let octokit = utils.testOctokit();
+  let REPO = "pytorch/pytorch";
+  let SHA = "random sha";
+
+  beforeEach(() => {
+    probot = utils.testProbot();
+    probot.load(triggerInductorTestsBot);
+  });
+
+  function mockRuns(
+    runs: { conclusion: string; created_at?: string; updated_at?: string }[]
+  ) {
+    return nock("https://api.github.com")
+      .get(`/repos/${REPO}/actions/runs?head_sha=${SHA}`)
+      .reply(200, {
+        workflow_runs: runs.map((run) => ({
+          event: "pull_request",
+          ...run,
+        })),
+      });
+  }
+
+  async function checkhasApprovedPullRunsReturns(value: boolean) {
+    expect(await hasApprovedPullRuns(octokit, "pytorch", "pytorch", SHA)).toBe(
+      value
+    );
+  }
+
+  afterEach(() => {
+    nock.cleanAll();
+    jest.restoreAllMocks();
+  });
+
+  test("successful runs = good", async () => {
+    mockRuns([{ conclusion: "success" }, { conclusion: "success" }]);
+    await checkhasApprovedPullRunsReturns(true);
+  });
+
+  test("at least 1 action required run = bad", async () => {
+    mockRuns([{ conclusion: "action_required" }, { conclusion: "success" }]);
+    await checkhasApprovedPullRunsReturns(false);
+  });
+
+  test("no runs = bad", async () => {
+    mockRuns([]);
+    await checkhasApprovedPullRunsReturns(false);
+  });
+
+  test("one startup failure = bad", async () => {
+    mockRuns([
+      {
+        conclusion: "failure",
+        created_at: "time",
+        updated_at: "time",
+      },
+    ]);
+    await checkhasApprovedPullRunsReturns(false);
+  });
+
+  test("one startup failure and one action required = bad", async () => {
+    mockRuns([
+      {
+        conclusion: "failure",
+        created_at: "time",
+        updated_at: "time",
+      },
+      { conclusion: "action_required" },
+    ]);
+    await checkhasApprovedPullRunsReturns(false);
+  });
+
+  test("one startup failure and one success = bad", async () => {
+    mockRuns([
+      {
+        conclusion: "failure",
+        created_at: "time",
+        updated_at: "time",
+      },
+      { conclusion: "success" },
+    ]);
+    await checkhasApprovedPullRunsReturns(false);
+  });
+});