add try and resume action for in-progress batches

nicktrn · nicktrn · commit 464902dc2aff · 2024-12-02T18:26:07.000Z
diff --git a/apps/webapp/app/components/runs/v3/CheckBatchCompletionDialog.tsx b/apps/webapp/app/components/runs/v3/CheckBatchCompletionDialog.tsx
@@ -0,0 +1,57 @@
+import { DialogClose } from "@radix-ui/react-dialog";
+import { Form, useNavigation } from "@remix-run/react";
+import { Button } from "~/components/primitives/Buttons";
+import { DialogContent, DialogHeader } from "~/components/primitives/Dialog";
+import { FormButtons } from "~/components/primitives/FormButtons";
+import { Paragraph } from "~/components/primitives/Paragraph";
+
+type CheckBatchCompletionDialogProps = {
+  batchId: string;
+  redirectPath: string;
+};
+
+export function CheckBatchCompletionDialog({
+  batchId,
+  redirectPath,
+}: CheckBatchCompletionDialogProps) {
+  const navigation = useNavigation();
+
+  const formAction = `/resources/batches/${batchId}/check-completion`;
+  const isLoading = navigation.formAction === formAction;
+
+  return (
+    <DialogContent key="check-completion">
+      <DialogHeader>Try and resume batch</DialogHeader>
+      <div className="flex flex-col gap-3 pt-3">
+        <Paragraph>
+          In rare cases, parent runs don't continue after child runs have completed.
+        </Paragraph>
+        <Paragraph>
+          If this doesn't help, please get in touch. We are working on a permanent fix for this.
+        </Paragraph>
+        <FormButtons
+          confirmButton={
+            <Form action={`/resources/batches/${batchId}/check-completion`} method="post">
+              <Button
+                type="submit"
+                name="redirectUrl"
+                value={redirectPath}
+                variant="primary/medium"
+                LeadingIcon={isLoading ? "spinner-white" : undefined}
+                disabled={isLoading}
+                shortcut={{ modifiers: ["meta"], key: "enter" }}
+              >
+                {isLoading ? "Attempting resume..." : "Attempt resume"}
+              </Button>
+            </Form>
+          }
+          cancelButton={
+            <DialogClose asChild>
+              <Button variant={"tertiary/medium"}>Cancel</Button>
+            </DialogClose>
+          }
+        />
+      </div>
+    </DialogContent>
+  );
+}
diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.batches/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.v3.$projectParam.batches/route.tsx
@@ -1,23 +1,26 @@
-import { ExclamationCircleIcon } from "@heroicons/react/20/solid";
+import { ArrowPathRoundedSquareIcon, ArrowRightIcon, CheckIcon, ExclamationCircleIcon } from "@heroicons/react/20/solid";
 import { BookOpenIcon } from "@heroicons/react/24/solid";
-import { useNavigation } from "@remix-run/react";
+import { useLocation, useNavigation } from "@remix-run/react";
 import { LoaderFunctionArgs } from "@remix-run/server-runtime";
 import { formatDuration } from "@trigger.dev/core/v3/utils/durations";
 import { typedjson, useTypedLoaderData } from "remix-typedjson";
 import { ListPagination } from "~/components/ListPagination";
 import { AdminDebugTooltip } from "~/components/admin/debugTooltip";
 import { EnvironmentLabel } from "~/components/environments/EnvironmentLabel";
 import { PageBody, PageContainer } from "~/components/layout/AppLayout";
-import { LinkButton } from "~/components/primitives/Buttons";
+import { Button, LinkButton } from "~/components/primitives/Buttons";
 import { DateTime } from "~/components/primitives/DateTime";
+import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
 import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
 import { Paragraph } from "~/components/primitives/Paragraph";
+import { PopoverMenuItem } from "~/components/primitives/Popover";
 import { Spinner } from "~/components/primitives/Spinner";
 import {
   Table,
   TableBlankRow,
   TableBody,
   TableCell,
+  TableCellMenu,
   TableHeader,
   TableHeaderCell,
   TableRow,
@@ -29,12 +32,17 @@ import {
   BatchStatusCombo,
   descriptionForBatchStatus,
 } from "~/components/runs/v3/BatchStatus";
+import { CheckBatchCompletionDialog } from "~/components/runs/v3/CheckBatchCompletionDialog";
 import { LiveTimer } from "~/components/runs/v3/LiveTimer";
 import { useOrganization } from "~/hooks/useOrganizations";
 import { useProject } from "~/hooks/useProject";
 import { redirectWithErrorMessage } from "~/models/message.server";
 import { findProjectBySlug } from "~/models/project.server";
-import { BatchList, BatchListPresenter } from "~/presenters/v3/BatchListPresenter.server";
+import {
+  BatchList,
+  BatchListItem,
+  BatchListPresenter,
+} from "~/presenters/v3/BatchListPresenter.server";
 import { requireUserId } from "~/services/session.server";
 import { docsPath, ProjectParamSchema, v3BatchRunsPath } from "~/utils/pathBuilder";
 
@@ -150,19 +158,22 @@ function BatchesTable({ batches, hasFilters, filters }: BatchList) {
           <TableHeaderCell>Duration</TableHeaderCell>
           <TableHeaderCell>Created</TableHeaderCell>
           <TableHeaderCell>Finished</TableHeaderCell>
+          <TableHeaderCell>
+            <span className="sr-only">Go to batch</span>
+          </TableHeaderCell>
         </TableRow>
       </TableHeader>
       <TableBody>
         {batches.length === 0 && !hasFilters ? (
-          <TableBlankRow colSpan={7}>
+          <TableBlankRow colSpan={8}>
             {!isLoading && (
               <div className="flex items-center justify-center">
                 <Paragraph className="w-auto">No batches</Paragraph>
               </div>
             )}
           </TableBlankRow>
         ) : batches.length === 0 ? (
-          <TableBlankRow colSpan={7}>
+          <TableBlankRow colSpan={8}>
             <div className="flex items-center justify-center">
               <Paragraph className="w-auto">No batches match these filters</Paragraph>
             </div>
@@ -215,13 +226,14 @@ function BatchesTable({ batches, hasFilters, filters }: BatchList) {
                 <TableCell to={path}>
                   {batch.finishedAt ? <DateTime date={batch.finishedAt} /> : "–"}
                 </TableCell>
+                <BatchActionsCell batch={batch} path={path} />
               </TableRow>
             );
           })
         )}
         {isLoading && (
           <TableBlankRow
-            colSpan={7}
+            colSpan={8}
             className="absolute left-0 top-0 flex h-full w-full items-center justify-center gap-2 bg-charcoal-900/90"
           >
             <Spinner /> <span className="text-text-dimmed">Loading…</span>
@@ -231,3 +243,50 @@ function BatchesTable({ batches, hasFilters, filters }: BatchList) {
     </Table>
   );
 }
+
+function BatchActionsCell({ batch, path }: { batch: BatchListItem; path: string }) {
+  const location = useLocation();
+
+  const isPending = batch.status === "PENDING";
+
+  if (!isPending) return <TableCell to={path}>{""}</TableCell>;
+
+  return (
+    <TableCellMenu
+      isSticky
+      popoverContent={
+        <>
+          <PopoverMenuItem
+            to={path}
+            icon={ArrowRightIcon}
+            leadingIconClassName="text-blue-500"
+            title="View batch"
+          />
+          {isPending && (
+            <Dialog>
+              <DialogTrigger
+                asChild
+                className="size-6 rounded-sm p-1 text-text-dimmed transition hover:bg-charcoal-700 hover:text-text-bright"
+              >
+                <Button
+                  variant="small-menu-item"
+                  LeadingIcon={ArrowPathRoundedSquareIcon}
+                  leadingIconClassName="text-success"
+                  fullWidth
+                  textAlignLeft
+                  className="w-full px-1.5 py-[0.9rem]"
+                >
+                  Try and resume
+                </Button>
+              </DialogTrigger>
+              <CheckBatchCompletionDialog
+                batchId={batch.id}
+                redirectPath={`${location.pathname}${location.search}`}
+              />
+            </Dialog>
+          )}
+        </>
+      }
+    />
+  );
+}
diff --git a/apps/webapp/app/routes/resources.batches.$batchId.check-completion.ts b/apps/webapp/app/routes/resources.batches.$batchId.check-completion.ts
@@ -0,0 +1,70 @@
+import { parse } from "@conform-to/zod";
+import { ActionFunction, json } from "@remix-run/node";
+import { assertExhaustive } from "@trigger.dev/core";
+import { z } from "zod";
+import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server";
+import { logger } from "~/services/logger.server";
+import { ResumeBatchRunService } from "~/v3/services/resumeBatchRun.server";
+
+export const checkCompletionSchema = z.object({
+  redirectUrl: z.string(),
+});
+
+const ParamSchema = z.object({
+  batchId: z.string(),
+});
+
+export const action: ActionFunction = async ({ request, params }) => {
+  const { batchId } = ParamSchema.parse(params);
+
+  const formData = await request.formData();
+  const submission = parse(formData, { schema: checkCompletionSchema });
+
+  if (!submission.value) {
+    return json(submission);
+  }
+
+  try {
+    const resumeBatchRunService = new ResumeBatchRunService();
+    const resumeResult = await resumeBatchRunService.call(batchId);
+
+    let message: string | undefined;
+
+    switch (resumeResult) {
+      case "ERROR": {
+        throw "Unknown error during batch completion check";
+      }
+      case "ALREADY_COMPLETED": {
+        message = "Batch already completed.";
+        break;
+      }
+      case "COMPLETED": {
+        message = "Batch completed and parent tasks resumed.";
+        break;
+      }
+      case "PENDING": {
+        message = "Child runs still in progress. Please try again later.";
+        break;
+      }
+      default: {
+        assertExhaustive(resumeResult);
+      }
+    }
+
+    return redirectWithSuccessMessage(submission.value.redirectUrl, request, message);
+  } catch (error) {
+    if (error instanceof Error) {
+      logger.error("Failed to check batch completion", {
+        error: {
+          name: error.name,
+          message: error.message,
+          stack: error.stack,
+        },
+      });
+      return redirectWithErrorMessage(submission.value.redirectUrl, request, error.message);
+    } else {
+      logger.error("Failed to check batch completion", { error });
+      return redirectWithErrorMessage(submission.value.redirectUrl, request, "Unknown error");
+    }
+  }
+};
diff --git a/apps/webapp/app/services/worker.server.ts b/apps/webapp/app/services/worker.server.ts
@@ -563,7 +563,7 @@ function getWorkerQueue() {
         handler: async (payload, job) => {
           const service = new ResumeBatchRunService();
 
-          return await service.call(payload.batchRunId);
+          await service.call(payload.batchRunId);
         },
       },
       "v3.resumeTaskDependency": {
diff --git a/apps/webapp/app/v3/services/resumeBatchRun.server.ts b/apps/webapp/app/v3/services/resumeBatchRun.server.ts
