Fix node options and OOM retries (#1897)

* don't hardcode max old space size in deployment images

* flags: treat underscores as hyphens

* append attempt number to runner name if >1

* improve retry spans for oom errors

Author: nicktrn

apps/supervisor/src/util.ts

@@ -5,6 +5,12 @@ export function getDockerHostDomain() {
   return isMacOs || isWindows ? "host.docker.internal" : "localhost";
 }
 
-export function getRunnerId(runId: string) {
-  return `runner-${runId.replace("run_", "")}`;
+export function getRunnerId(runId: string, attemptNumber?: number) {
+  const parts = ["runner", runId.replace("run_", "")];
+
+  if (attemptNumber && attemptNumber > 1) {
+    parts.push(`attempt-${attemptNumber}`);
+  }
+
+  return parts.join("-");
 }

apps/supervisor/src/workloadManager/docker.ts

@@ -22,7 +22,7 @@ export class DockerWorkloadManager implements WorkloadManager {
   async create(opts: WorkloadManagerCreateOptions) {
     this.logger.log("[DockerWorkloadProvider] Creating container", { opts });
 
-    const runnerId = getRunnerId(opts.runFriendlyId);
+    const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
 
     const runArgs = [
       "run",

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -31,7 +31,7 @@ export class KubernetesWorkloadManager implements WorkloadManager {
   async create(opts: WorkloadManagerCreateOptions) {
     this.logger.log("[KubernetesWorkloadManager] Creating container", { opts });
 
-    const runnerId = getRunnerId(opts.runFriendlyId);
+    const runnerId = getRunnerId(opts.runFriendlyId, opts.nextAttemptNumber);
 
     try {
       await this.k8s.core.createNamespacedPod({

apps/webapp/app/v3/runEngineHandlers.server.ts

@@ -326,12 +326,19 @@ export function registerRunEngineEventBusHandlers() {
 
   engine.eventBus.on("runRetryScheduled", async ({ time, run, environment, retryAt }) => {
     try {
-      await eventRepository.recordEvent(`Retry #${run.attemptNumber} delay`, {
+      let retryMessage = `Retry #${run.attemptNumber} delay`;
+
+      if (run.nextMachineAfterOOM) {
+        retryMessage += ` after OOM`;
+      }
+
+      await eventRepository.recordEvent(retryMessage, {
         taskSlug: run.taskIdentifier,
         environment,
         attributes: {
           properties: {
             retryAt: retryAt.toISOString(),
+            nextMachine: run.nextMachineAfterOOM,
           },
           runId: run.friendlyId,
           style: {

internal-packages/run-engine/src/engine/eventBus.ts

@@ -85,6 +85,7 @@ export type EventBusEvents = {
         traceContext: Record<string, string | undefined>;
         taskIdentifier: string;
         baseCostInCents: number;
+        nextMachineAfterOOM?: string;
       };
       organization: {
         id: string;

internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts

@@ -692,6 +692,7 @@ export class RunAttemptSystem {
                   traceContext: run.traceContext as Record<string, string | undefined>,
                   baseCostInCents: run.baseCostInCents,
                   spanId: run.spanId,
+                  nextMachineAfterOOM: retryResult.machine,
                 },
                 organization: {
                   id: run.runtimeEnvironment.organizationId,

packages/cli-v3/src/deploy/buildImage.ts

@@ -688,8 +688,7 @@ ENV TRIGGER_PROJECT_ID=\${TRIGGER_PROJECT_ID} \
     TRIGGER_CONTENT_HASH=\${TRIGGER_CONTENT_HASH} \
     TRIGGER_PROJECT_REF=\${TRIGGER_PROJECT_REF} \
     NODE_EXTRA_CA_CERTS=\${NODE_EXTRA_CA_CERTS} \
-    NODE_ENV=production \
-    NODE_OPTIONS="--max_old_space_size=8192"
+    NODE_ENV=production
 
 # Copy the files from the install stage
 COPY --from=build --chown=node:node /app ./

packages/core/src/v3/build/flags.test.ts

@@ -18,6 +18,11 @@ describe("dedupFlags", () => {
     expect(dedupFlags("--log=info --log=warn --log=error")).toBe("--log=error");
   });
 
+  it("should treat underscores as hyphens", () => {
+    expect(dedupFlags("--debug_level=info")).toBe("--debug-level=info");
+    expect(dedupFlags("--debug_level=info --debug-level=warn")).toBe("--debug-level=warn");
+  });
+
   it("should handle mix of flags with and without values", () => {
     expect(dedupFlags("--debug=false -v --debug=true")).toBe("-v --debug=true");
     expect(dedupFlags("-v --quiet -v")).toBe("--quiet -v");

packages/core/src/v3/build/flags.ts

@@ -27,11 +27,11 @@ export function dedupFlags(flags: string): string {
     .map((flag): [string, string | boolean] => {
       const equalIndex = flag.indexOf("=");
       if (equalIndex !== -1) {
-        const key = flag.slice(0, equalIndex);
+        const key = flag.slice(0, equalIndex).replace(/_/g, "-");
         const value = flag.slice(equalIndex + 1);
         return [key, value];
       } else {
-        return [flag, true];
+        return [flag.replace(/_/g, "-"), true];
       }
     });