More fixes

Author: rculbertson

modal-js/src/function.ts

@@ -12,14 +12,17 @@ import type { LookupOptions } from "./app";
 import { client } from "./client";
 import { FunctionCall } from "./function_call";
 import { environmentName } from "./config";
-import { NotFoundError } from "./errors";
+import { InternalFailure, NotFoundError } from "./errors";
 import { dumps } from "./pickle";
 import { ClientError, Status } from "nice-grpc";
 import { ControlPlaneInvocation } from "./invocation";
 
 // From: modal/_utils/blob_utils.py
 const maxObjectSizeBytes = 2 * 1024 * 1024; // 2 MiB
 
+// From: client/modal/_functions.py
+const maxSystemRetries = 8;
+
 /** Represents a deployed Modal Function, which can be invoked remotely. */
 export class Function_ {
   readonly functionId: string;
@@ -62,7 +65,19 @@ export class Function_ {
       input,
       FunctionCallInvocationType.FUNCTION_CALL_INVOCATION_TYPE_SYNC,
     );
-    return await invocation.await();
+    let retryCount = 0;
+    while (true) {
+      try {
+        return await invocation.await();
+      } catch (err) {
+        if (err instanceof InternalFailure && retryCount <= maxSystemRetries) {
+          await invocation.retry(retryCount);
+          retryCount++;
+        } else {
+          throw err;
+        }
+      }
+    }
   }
 
   // Spawn a single input into a remote function.

modal-js/src/invocation.ts

@@ -30,7 +30,7 @@ export interface Invocation {
    */
   await(): Promise<any>;
 
-  retry(): Promise<void>;
+  retry(retryCount: number): Promise<void>;
 }
 
 /**
@@ -41,7 +41,6 @@ export class ControlPlaneInvocation implements Invocation {
   private readonly input?: FunctionInput;
   private readonly functionCallJwt?: string;
   private inputJwt?: string;
-  private retryCount: number = 0;
 
   private constructor(
     functionCallId: string,
@@ -81,7 +80,7 @@ export class ControlPlaneInvocation implements Invocation {
     return await pollControlPlaneForOutput(this.functionCallId, timeout);
   }
 
-  async retry(): Promise<void> {
+  async retry(retryCount: number): Promise<void> {
     // we do not expect this to happen
     if (!this.input) {
       throw new Error("Cannot retry function invocation - input missing");
@@ -90,15 +89,14 @@ export class ControlPlaneInvocation implements Invocation {
     const retryItem: FunctionRetryInputsItem = {
       inputJwt: this.inputJwt!,
       input: this.input,
-      retryCount: this.retryCount,
+      retryCount: retryCount,
     };
 
     const functionRetryResponse = await client.functionRetryInputs({
       functionCallJwt: this.functionCallJwt,
       inputs: [retryItem],
     });
     this.inputJwt = functionRetryResponse.inputJwts[0];
-    this.retryCount += 1;
   }
 
   private static async execFunctionCall(