Feat/retry 429 functions

package.json

@@ -127,6 +127,7 @@
     "deep-equal-in-any-order": "^2.0.6",
     "exegesis": "^4.2.0",
     "exegesis-express": "^4.0.0",
+    "exponential-backoff": "^3.1.2",
     "express": "^4.16.4",
     "filesize": "^6.1.0",
     "form-data": "^4.0.1",

src/gcp/cloudfunctions.ts

@@ -18,6 +18,7 @@ import {
   CODEBASE_LABEL,
   HASH_LABEL,
 } from "../functions/constants";
+import { with429Backoff } from "./retry429";
 
 export const API_VERSION = "v1";
 const client = new Client({ urlPrefix: functionsOrigin(), apiVersion: API_VERSION });
@@ -209,10 +210,8 @@ export async function generateUploadUrl(projectId: string, location: string): Pr
   const endpoint = `/${parent}/functions:generateUploadUrl`;
 
   try {
-    const res = await client.post<unknown, { uploadUrl: string }>(
-      endpoint,
-      {},
-      { retryCodes: [503] },
+    const res = await with429Backoff("generateUploadUrl", location, () =>
+      client.post<unknown, { uploadUrl: string }>(endpoint, {}, { retryCodes: [503] }),
     );
     return res.body.uploadUrl;
   } catch (err: any) {
@@ -241,9 +240,8 @@ export async function createFunction(
   };
 
   try {
-    const res = await client.post<Omit<CloudFunction, OutputOnlyFields>, CloudFunction>(
-      endpoint,
-      cloudFunction,
+    const res = await with429Backoff("create", cloudFunction.name, () =>
+      client.post<Omit<CloudFunction, OutputOnlyFields>, CloudFunction>(endpoint, cloudFunction),
     );
     return {
       name: res.body.name,
@@ -405,14 +403,12 @@ export async function updateFunction(
   // Failure policy is always an explicit policy and is only signified by the presence or absence of
   // a protobuf.Empty value, so we have to manually add it in the missing case.
   try {
-    const res = await client.patch<Omit<CloudFunction, OutputOnlyFields>, CloudFunction>(
-      endpoint,
-      cloudFunction,
-      {
+    const res = await with429Backoff("update", cloudFunction.name, () =>
+      client.patch<Omit<CloudFunction, OutputOnlyFields>, CloudFunction>(endpoint, cloudFunction, {
         queryParams: {
           updateMask: fieldMasks.join(","),
         },
-      },
+      }),
     );
     return {
       done: false,
@@ -431,7 +427,7 @@ export async function updateFunction(
 export async function deleteFunction(name: string): Promise<Operation> {
   const endpoint = `/${name}`;
   try {
-    const res = await client.delete<Operation>(endpoint);
+    const res = await with429Backoff("delete", name, () => client.delete<Operation>(endpoint));
     return {
       done: false,
       name: res.body.name,

src/gcp/cloudfunctionsv2.ts

@@ -19,6 +19,7 @@ import {
 import { RequireKeys } from "../metaprogramming";
 import { captureRuntimeValidationError } from "./cloudfunctions";
 import { mebibytes } from "./k8s";
+import { with429Backoff } from "./retry429";
 
 export const API_VERSION = "v2";
 
@@ -273,8 +274,10 @@ export async function generateUploadUrl(
   location: string,
 ): Promise<GenerateUploadUrlResponse> {
   try {
-    const res = await client.post<never, GenerateUploadUrlResponse>(
-      `projects/${projectId}/locations/${location}/functions:generateUploadUrl`,
+    const res = await with429Backoff("generateUploadUrl", `${projectId}/${location}`, () =>
+      client.post<never, GenerateUploadUrlResponse>(
+        `projects/${projectId}/locations/${location}/functions:generateUploadUrl`,
+      ),
     );
     return res.body;
   } catch (err: any) {
@@ -308,10 +311,10 @@ export async function createFunction(cloudFunction: InputCloudFunction): Promise
   };
 
   try {
-    const res = await client.post<typeof cloudFunction, Operation>(
-      components.join("/"),
-      cloudFunction,
-      { queryParams: { functionId } },
+    const res = await with429Backoff("create", cloudFunction.name, () =>
+      client.post<typeof cloudFunction, Operation>(components.join("/"), cloudFunction, {
+        queryParams: { functionId },
+      }),
     );
     return res.body;
   } catch (err: any) {
@@ -399,13 +402,11 @@ export async function updateFunction(cloudFunction: InputCloudFunction): Promise
   );
 
   try {
-    const queryParams = {
-      updateMask: fieldMasks.join(","),
-    };
-    const res = await client.patch<typeof cloudFunction, Operation>(
-      cloudFunction.name,
-      cloudFunction,
-      { queryParams },
+    const queryParams = { updateMask: fieldMasks.join(",") };
+    const res = await with429Backoff("update", cloudFunction.name, () =>
+      client.patch<typeof cloudFunction, Operation>(cloudFunction.name, cloudFunction, {
+        queryParams,
+      }),
     );
     return res.body;
   } catch (err: any) {
@@ -419,7 +420,9 @@ export async function updateFunction(cloudFunction: InputCloudFunction): Promise
  */
 export async function deleteFunction(cloudFunction: string): Promise<Operation> {
   try {
-    const res = await client.delete<Operation>(cloudFunction);
+    const res = await with429Backoff("delete", cloudFunction, () =>
+      client.delete<Operation>(cloudFunction),
+    );
     return res.body;
   } catch (err: any) {
     throw functionsOpLogReject({ name: cloudFunction } as InputCloudFunction, "update", err);

src/gcp/retry429.ts

@@ -0,0 +1,23 @@
+import * as clc from "colorette";
+import * as utils from "../utils";
+import { withHttpBackoff } from "./retryWithBackoff";
+
+/**
+ * Convenience wrapper for Cloud Functions deploy operations.
+ * Retries **only** on HTTP 429 and logs a clear message.
+ */
+export function with429Backoff<T>(
+  op: "create" | "update" | "delete" | "generateUploadUrl",
+  resourceName: string,
+  thunk: () => Promise<T>,
+) {
+  return withHttpBackoff(thunk, {
+    statuses: [429],
+    onRetry: ({ attempt, maxAttempts }) => {
+      utils.logLabeledWarning(
+        "functions",
+        `${clc.bold(clc.yellow("429 (Quota Exceeded)"))} on ${op} ${resourceName}; retrying (attempt ${attempt}${maxAttempts ? `/${maxAttempts}` : ""})…`,
+      );
+    },
+  });
+}

src/gcp/retryWithBackoff.ts

@@ -0,0 +1,82 @@
+import { backOff, type BackoffOptions } from "exponential-backoff";
+
+/** Extract an HTTP status from common firebase-tools error shapes. */
+export const statusOf = (err: any): number | undefined =>
+  err?.context?.response?.statusCode ?? err?.status ?? err?.statusCode;
+
+export interface HttpBackoffOptions extends Partial<BackoffOptions> {
+  /**
+   * Optional: custom predicate to decide if an error should be retried.
+   * If provided, this takes precedence over `statuses`.
+   */
+  shouldRetry?: (err: unknown, nextAttempt: number) => boolean;
+
+  /**
+   * Optional: retry when HTTP status is in this set (e.g., [429, 503]).
+   * Ignored if `shouldRetry` is provided.
+   */
+  statuses?: readonly number[] | ReadonlySet<number>;
+
+  /**
+   * Optional: called right before a retry. Use e.g. to log context.
+   */
+  onRetry?: (info: {
+    err: unknown;
+    attempt: number;
+    maxAttempts?: number;
+    status?: number;
+  }) => void;
+}
+
+/** Base backoff knobs (env-tunable if you want to tweak in CI). */
+const defaultBackoff: BackoffOptions = {
+  numOfAttempts: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_MAX ?? 9),
+  startingDelay: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_BASE_MS ?? 60 * 1_000),
+  timeMultiple: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_MULTIPLIER ?? 2),
+  maxDelay: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_MAX_MS ?? 20 * 60 * 1_000),
+  jitter: "full",
+};
+
+/**
+ * Generic exponential backoff wrapper for HTTP-ish operations.
+ * - Retries when `shouldRetry` returns true OR status is in `statuses`.
+ * - If neither is provided, nothing will retry.
+ */
+export async function withHttpBackoff<T>(
+  thunk: () => Promise<T>,
+  opts: HttpBackoffOptions = {},
+): Promise<T> {
+  const statuses =
+    opts.statuses instanceof Set
+      ? opts.statuses
+      : Array.isArray(opts.statuses)
+        ? new Set<number>(opts.statuses)
+        : undefined;
+
+  const options = {
+    ...defaultBackoff,
+    ...opts,
+  };
+
+  return backOff(thunk, {
+    ...options,
+    retry: (err, nextAttempt) => {
+      const st = statusOf(err);
+      const shouldRetry =
+        opts.shouldRetry?.(err, nextAttempt) ??
+        (st !== undefined && statuses ? statuses.has(st) : false);
+
+      if (shouldRetry) {
+        if (opts.onRetry) {
+          opts.onRetry({
+            err,
+            attempt: nextAttempt,
+            maxAttempts: options.numOfAttempts,
+            status: st,
+          });
+        }
+      }
+      return shouldRetry;
+    },
+  });
+}