improve retry spans for oom errors

Author: nicktrn

apps/webapp/app/v3/runEngineHandlers.server.ts

@@ -326,12 +326,19 @@ export function registerRunEngineEventBusHandlers() {
 
   engine.eventBus.on("runRetryScheduled", async ({ time, run, environment, retryAt }) => {
     try {
-      await eventRepository.recordEvent(`Retry #${run.attemptNumber} delay`, {
+      let retryMessage = `Retry #${run.attemptNumber} delay`;
+
+      if (run.nextMachineAfterOOM) {
+        retryMessage += ` after OOM`;
+      }
+
+      await eventRepository.recordEvent(retryMessage, {
         taskSlug: run.taskIdentifier,
         environment,
         attributes: {
           properties: {
             retryAt: retryAt.toISOString(),
+            nextMachine: run.nextMachineAfterOOM,
           },
           runId: run.friendlyId,
           style: {

internal-packages/run-engine/src/engine/eventBus.ts

@@ -85,6 +85,7 @@ export type EventBusEvents = {
         traceContext: Record<string, string | undefined>;
         taskIdentifier: string;
         baseCostInCents: number;
+        nextMachineAfterOOM?: string;
       };
       organization: {
         id: string;

internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts

@@ -692,6 +692,7 @@ export class RunAttemptSystem {
                   traceContext: run.traceContext as Record<string, string | undefined>,
                   baseCostInCents: run.baseCostInCents,
                   spanId: run.spanId,
+                  nextMachineAfterOOM: retryResult.machine,
                 },
                 organization: {
                   id: run.runtimeEnvironment.organizationId,