Only retry API failures (#7831)

* Only retry API failures

* fix lint

Author: penalosa

packages/wrangler/src/__tests__/versions/versions.upload.test.ts

@@ -169,5 +169,7 @@ describe("versions upload", () => {
 			Uploaded test-name (TIMINGS)
 			Worker Version ID: 51e4886e-2db7-4900-8d38-fbfecfeab993"
 		`);
+
+		expect(std.info).toContain("Retrying API call after error...");
 	});
 });

packages/wrangler/src/deploy/deploy.ts

@@ -49,7 +49,7 @@ import {
 } from "../sourcemap";
 import triggersDeploy from "../triggers/deploy";
 import { printBindings } from "../utils/print-bindings";
-import { retryOnError } from "../utils/retry";
+import { retryOnAPIFailure } from "../utils/retry";
 import {
 	createDeployment,
 	patchNonVersionedScriptSettings,
@@ -826,7 +826,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 				// If we're using the new APIs, first upload the version
 				if (canUseNewVersionsDeploymentsApi) {
 					// Upload new version
-					const versionResult = await retryOnError(async () =>
+					const versionResult = await retryOnAPIFailure(async () =>
 						fetchResult<ApiVersion>(
 							`/accounts/${accountId}/workers/scripts/${scriptName}/versions`,
 							{
@@ -861,7 +861,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 						startup_time_ms: versionResult.startup_time_ms,
 					};
 				} else {
-					result = await retryOnError(async () =>
+					result = await retryOnAPIFailure(async () =>
 						fetchResult<{
 							id: string | null;
 							etag: string | null;

packages/wrangler/src/parse.ts

@@ -88,6 +88,10 @@ export class APIError extends ParseError {
 		return false;
 	}
 
+	isRetryable() {
+		return String(this.#status).startsWith("5");
+	}
+
 	// Allow `APIError`s to be marked as handled.
 	#reportable = true;
 	get reportable() {

packages/wrangler/src/pipelines/index.ts

@@ -5,7 +5,7 @@ import { logger } from "../logger";
 import * as metrics from "../metrics";
 import { APIError } from "../parse";
 import { requireAuth } from "../user";
-import { retryOnError } from "../utils/retry";
+import { retryOnAPIFailure } from "../utils/retry";
 import { printWranglerBanner } from "../wrangler-banner";
 import {
 	createPipeline,
@@ -63,7 +63,7 @@ async function authorizeR2Bucket(
 
 	// Wait for token to settle/propagate, retry up to 10 times, with 1s waits in-between errors
 	!__testSkipDelaysFlag &&
-		(await retryOnError(
+		(await retryOnAPIFailure(
 			async () => {
 				await r2.send(
 					new HeadBucketCommand({

packages/wrangler/src/utils/retry.ts

@@ -1,18 +1,45 @@
 import { setTimeout } from "node:timers/promises";
+import chalk from "chalk";
+import { logger } from "../logger";
+import { APIError } from "../parse";
 
-export async function retryOnError<T>(
+const MAX_ATTEMPTS = 3;
+/**
+ * Wrap around calls to the Cloudflare API to automatically retry
+ * calls that result in a 5xx error code, indicating an API failure.
+ *
+ * Retries will back off at a rate of 1000ms per retry, with a 0ms delay for the first retry
+ *
+ * Note: this will not retry 4xx or other failures, as those are
+ * likely legitimate user error.
+ */
+export async function retryOnAPIFailure<T>(
 	action: () => T | Promise<T>,
-	backoff = 2_000,
-	attempts = 3
+	backoff = 0,
+	attempts = MAX_ATTEMPTS
 ): Promise<T> {
 	try {
 		return await action();
 	} catch (err) {
+		if (
+			(err instanceof APIError && !err.isRetryable()) ||
+			!(err instanceof TypeError)
+		) {
+			throw err;
+		}
+
+		logger.info(chalk.dim(`Retrying API call after error...`));
+		logger.debug(err);
+
 		if (attempts <= 1) {
 			throw err;
 		}
 
 		await setTimeout(backoff);
-		return retryOnError(action, backoff, attempts - 1);
+		return retryOnAPIFailure(
+			action,
+			backoff + (MAX_ATTEMPTS - attempts) * 1000,
+			attempts - 1
+		);
 	}
 }

packages/wrangler/src/versions/upload.ts

@@ -53,7 +53,7 @@ import { getRules } from "../utils/getRules";
 import { getScriptName } from "../utils/getScriptName";
 import { isLegacyEnv } from "../utils/isLegacyEnv";
 import { printBindings } from "../utils/print-bindings";
-import { retryOnError } from "../utils/retry";
+import { retryOnAPIFailure } from "../utils/retry";
 import type { AssetsOptions } from "../assets";
 import type { Config } from "../config";
 import type { Rule } from "../config/environment";
@@ -741,7 +741,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 			try {
 				const body = createWorkerUploadForm(worker);
 
-				const result = await retryOnError(async () =>
+				const result = await retryOnAPIFailure(async () =>
 					fetchResult<{
 						id: string;
 						startup_time_ms: number;