diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 981a5573eb3..209d0772653 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -34,6 +34,7 @@ "deep-equal-in-any-order": "^2.0.6", "exegesis": "^4.2.0", "exegesis-express": "^4.0.0", + "exponential-backoff": "^3.1.2", "express": "^4.16.4", "filesize": "^6.1.0", "form-data": "^4.0.1", @@ -9890,6 +9891,12 @@ } } }, + "node_modules/exponential-backoff": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.2.tgz", + "integrity": "sha512-8QxYTVXUkuy7fIIoitQkPwGonB8F3Zj8eEO8Sqg9Zv/bkI7RJAzowee4gr81Hak/dUTpA2Z7VfQgoijjPNlUZA==", + "license": "Apache-2.0" + }, "node_modules/express": { "version": "4.21.2", "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", @@ -29029,6 +29036,11 @@ "exegesis": "^4.1.0" } }, + "exponential-backoff": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.2.tgz", + "integrity": "sha512-8QxYTVXUkuy7fIIoitQkPwGonB8F3Zj8eEO8Sqg9Zv/bkI7RJAzowee4gr81Hak/dUTpA2Z7VfQgoijjPNlUZA==" + }, "express": { "version": "4.21.2", "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", diff --git a/package.json b/package.json index 30d7d498069..ab0576f74a1 100644 --- a/package.json +++ b/package.json @@ -127,6 +127,7 @@ "deep-equal-in-any-order": "^2.0.6", "exegesis": "^4.2.0", "exegesis-express": "^4.0.0", + "exponential-backoff": "^3.1.2", "express": "^4.16.4", "filesize": "^6.1.0", "form-data": "^4.0.1", diff --git a/src/gcp/cloudfunctions.ts b/src/gcp/cloudfunctions.ts index c3e5e3fd8fe..ed685b663bc 100644 --- a/src/gcp/cloudfunctions.ts +++ b/src/gcp/cloudfunctions.ts @@ -18,6 +18,7 @@ import { CODEBASE_LABEL, HASH_LABEL, } from "../functions/constants"; +import { with429Backoff } from "./retry429"; export const API_VERSION = "v1"; const client = new Client({ urlPrefix: functionsOrigin(), apiVersion: API_VERSION }); @@ -209,10 +210,8 @@ export async function generateUploadUrl(projectId: string, location: string): Pr const endpoint = `/${parent}/functions:generateUploadUrl`; try { - const res = await client.post( - endpoint, - {}, - { retryCodes: [503] }, + const res = await with429Backoff("generateUploadUrl", location, () => + client.post(endpoint, {}, { retryCodes: [503] }), ); return res.body.uploadUrl; } catch (err: any) { @@ -241,9 +240,8 @@ export async function createFunction( }; try { - const res = await client.post, CloudFunction>( - endpoint, - cloudFunction, + const res = await with429Backoff("create", cloudFunction.name, () => + client.post, CloudFunction>(endpoint, cloudFunction), ); return { name: res.body.name, @@ -405,14 +403,12 @@ export async function updateFunction( // Failure policy is always an explicit policy and is only signified by the presence or absence of // a protobuf.Empty value, so we have to manually add it in the missing case. try { - const res = await client.patch, CloudFunction>( - endpoint, - cloudFunction, - { + const res = await with429Backoff("update", cloudFunction.name, () => + client.patch, CloudFunction>(endpoint, cloudFunction, { queryParams: { updateMask: fieldMasks.join(","), }, - }, + }), ); return { done: false, @@ -431,7 +427,7 @@ export async function updateFunction( export async function deleteFunction(name: string): Promise { const endpoint = `/${name}`; try { - const res = await client.delete(endpoint); + const res = await with429Backoff("delete", name, () => client.delete(endpoint)); return { done: false, name: res.body.name, diff --git a/src/gcp/cloudfunctionsv2.ts b/src/gcp/cloudfunctionsv2.ts index 37291f037c2..21a870556bf 100644 --- a/src/gcp/cloudfunctionsv2.ts +++ b/src/gcp/cloudfunctionsv2.ts @@ -19,6 +19,7 @@ import { import { RequireKeys } from "../metaprogramming"; import { captureRuntimeValidationError } from "./cloudfunctions"; import { mebibytes } from "./k8s"; +import { with429Backoff } from "./retry429"; export const API_VERSION = "v2"; @@ -273,8 +274,10 @@ export async function generateUploadUrl( location: string, ): Promise { try { - const res = await client.post( - `projects/${projectId}/locations/${location}/functions:generateUploadUrl`, + const res = await with429Backoff("generateUploadUrl", `${projectId}/${location}`, () => + client.post( + `projects/${projectId}/locations/${location}/functions:generateUploadUrl`, + ), ); return res.body; } catch (err: any) { @@ -308,10 +311,10 @@ export async function createFunction(cloudFunction: InputCloudFunction): Promise }; try { - const res = await client.post( - components.join("/"), - cloudFunction, - { queryParams: { functionId } }, + const res = await with429Backoff("create", cloudFunction.name, () => + client.post(components.join("/"), cloudFunction, { + queryParams: { functionId }, + }), ); return res.body; } catch (err: any) { @@ -399,13 +402,11 @@ export async function updateFunction(cloudFunction: InputCloudFunction): Promise ); try { - const queryParams = { - updateMask: fieldMasks.join(","), - }; - const res = await client.patch( - cloudFunction.name, - cloudFunction, - { queryParams }, + const queryParams = { updateMask: fieldMasks.join(",") }; + const res = await with429Backoff("update", cloudFunction.name, () => + client.patch(cloudFunction.name, cloudFunction, { + queryParams, + }), ); return res.body; } catch (err: any) { @@ -419,7 +420,9 @@ export async function updateFunction(cloudFunction: InputCloudFunction): Promise */ export async function deleteFunction(cloudFunction: string): Promise { try { - const res = await client.delete(cloudFunction); + const res = await with429Backoff("delete", cloudFunction, () => + client.delete(cloudFunction), + ); return res.body; } catch (err: any) { throw functionsOpLogReject({ name: cloudFunction } as InputCloudFunction, "update", err); diff --git a/src/gcp/retry429.ts b/src/gcp/retry429.ts new file mode 100644 index 00000000000..b586a528f8c --- /dev/null +++ b/src/gcp/retry429.ts @@ -0,0 +1,23 @@ +import * as clc from "colorette"; +import * as utils from "../utils"; +import { withHttpBackoff } from "./retryWithBackoff"; + +/** + * Convenience wrapper for Cloud Functions deploy operations. + * Retries **only** on HTTP 429 and logs a clear message. + */ +export function with429Backoff( + op: "create" | "update" | "delete" | "generateUploadUrl", + resourceName: string, + thunk: () => Promise, +) { + return withHttpBackoff(thunk, { + statuses: [429], + onRetry: ({ attempt, maxAttempts }) => { + utils.logLabeledWarning( + "functions", + `${clc.bold(clc.yellow("429 (Quota Exceeded)"))} on ${op} ${resourceName}; retrying (attempt ${attempt}${maxAttempts ? `/${maxAttempts}` : ""})…`, + ); + }, + }); +} diff --git a/src/gcp/retryWithBackoff.ts b/src/gcp/retryWithBackoff.ts new file mode 100644 index 00000000000..4ce51751563 --- /dev/null +++ b/src/gcp/retryWithBackoff.ts @@ -0,0 +1,82 @@ +import { backOff, type BackoffOptions } from "exponential-backoff"; + +/** Extract an HTTP status from common firebase-tools error shapes. */ +export const statusOf = (err: any): number | undefined => + err?.context?.response?.statusCode ?? err?.status ?? err?.statusCode; + +export interface HttpBackoffOptions extends Partial { + /** + * Optional: custom predicate to decide if an error should be retried. + * If provided, this takes precedence over `statuses`. + */ + shouldRetry?: (err: unknown, nextAttempt: number) => boolean; + + /** + * Optional: retry when HTTP status is in this set (e.g., [429, 503]). + * Ignored if `shouldRetry` is provided. + */ + statuses?: readonly number[] | ReadonlySet; + + /** + * Optional: called right before a retry. Use e.g. to log context. + */ + onRetry?: (info: { + err: unknown; + attempt: number; + maxAttempts?: number; + status?: number; + }) => void; +} + +/** Base backoff knobs (env-tunable if you want to tweak in CI). */ +const defaultBackoff: BackoffOptions = { + numOfAttempts: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_MAX ?? 9), + startingDelay: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_BASE_MS ?? 60 * 1_000), + timeMultiple: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_MULTIPLIER ?? 2), + maxDelay: Number(process.env.FIREBASE_TOOLS_FUNC_RETRY_MAX_MS ?? 20 * 60 * 1_000), + jitter: "full", +}; + +/** + * Generic exponential backoff wrapper for HTTP-ish operations. + * - Retries when `shouldRetry` returns true OR status is in `statuses`. + * - If neither is provided, nothing will retry. + */ +export async function withHttpBackoff( + thunk: () => Promise, + opts: HttpBackoffOptions = {}, +): Promise { + const statuses = + opts.statuses instanceof Set + ? opts.statuses + : Array.isArray(opts.statuses) + ? new Set(opts.statuses) + : undefined; + + const options = { + ...defaultBackoff, + ...opts, + }; + + return backOff(thunk, { + ...options, + retry: (err, nextAttempt) => { + const st = statusOf(err); + const shouldRetry = + opts.shouldRetry?.(err, nextAttempt) ?? + (st !== undefined && statuses ? statuses.has(st) : false); + + if (shouldRetry) { + if (opts.onRetry) { + opts.onRetry({ + err, + attempt: nextAttempt, + maxAttempts: options.numOfAttempts, + status: st, + }); + } + } + return shouldRetry; + }, + }); +}