feat: Add exponential backoff to requests

- Wrapped api call function with exponential backoff function, which will retry calls that return either 429 - too many requests code or http 500+ codes.
- Added test for the exp backoff

Author: JanHranicky

nodes/Apify/__tests__/Apify.node.spec.ts

@@ -509,4 +509,27 @@ describe('Apify Node', () => {
 			});
 		});
 	});
+
+	describe('api calls', () => {
+		it('should retry the specified number of times with exponential delays', async () => {
+			const storeId = 'yTfMu13hDFe9bRjx6';
+			const recordKey = 'INPUT';
+
+			const scope = nock('https://api.apify.com')
+				.get(`/v2/key-value-stores/${storeId}/records/${recordKey}`)
+				.reply(500)
+				.get(`/v2/key-value-stores/${storeId}/records/${recordKey}`)
+				.reply(429)
+				.get(`/v2/key-value-stores/${storeId}/records/${recordKey}`)
+				.reply(200);
+
+			const getKeyValueStoreRecordWorkflow = require('./workflows/key-value-stores/get-key-value-store-record.workflow.json');
+			await executeWorkflow({
+				credentialsHelper,
+				workflow: getKeyValueStoreRecordWorkflow,
+			});
+
+			expect(scope.isDone()).toBe(true);
+		});
+	});
 });

nodes/Apify/helpers/consts.ts

@@ -14,3 +14,7 @@ export const memoryOptions = [
 	{ name: '16384 MB (16 GB)', value: 16384 },
 	{ name: '32768 MB (32 GB)', value: 32768 },
 ];
+
+export const DEFAULT_EXP_BACKOFF_INTERVAL = 1;
+export const DEFAULT_EXP_BACKOFF_EXPONENTIAL = 2;
+export const DEFAULT_EXP_BACKOFF_RETRIES = 5;

nodes/Apify/resources/genericFunctions.ts

@@ -8,6 +8,11 @@ import {
 	type ILoadOptionsFunctions,
 	type IRequestOptions,
 } from 'n8n-workflow';
+import {
+	DEFAULT_EXP_BACKOFF_EXPONENTIAL,
+	DEFAULT_EXP_BACKOFF_INTERVAL,
+	DEFAULT_EXP_BACKOFF_RETRIES,
+} from '../helpers/consts';
 
 type IApiRequestOptions = IRequestOptions & { uri?: string };
 
@@ -50,7 +55,9 @@ export async function apiRequest(
 			);
 		}
 
-		return await this.helpers.requestWithAuthentication.call(this, authenticationMethod, options);
+		return await retryWithExponentialBackoff(() =>
+			this.helpers.requestWithAuthentication.call(this, authenticationMethod, options),
+		);
 	} catch (error) {
 		/**
 		 * using `error instanceof NodeApiError` results in `false`
@@ -71,6 +78,60 @@ export async function apiRequest(
 	}
 }
 
+/**
+ * Checks if the given status code is retryable
+ * Status codes 429 (rate limit) and 500+ are retried,
+ * Other status codes 300-499 (except 429) are not retried,
+ * because the error is probably caused by invalid URL (redirect 3xx) or invalid user input (4xx).
+ */
+function isStatusCodeRetryable(statusCode: number) {
+	if (Number.isNaN(statusCode)) return false;
+
+	const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429;
+	const isRateLimitError = statusCode === RATE_LIMIT_EXCEEDED_STATUS_CODE;
+	const isInternalError = statusCode >= 500;
+	return isRateLimitError || isInternalError;
+}
+
+/**
+ * Wraps a function with exponential backoff.
+ * If request fails with http code 500+ or doesn't return
+ * a code at all it is retried in 1s,2s,4s,.. up to maxRetries
+ * @param fn
+ * @param interval
+ * @param exponential
+ * @param maxRetries
+ * @returns
+ */
+export async function retryWithExponentialBackoff(
+	fn: () => Promise<any>,
+	interval: number = DEFAULT_EXP_BACKOFF_INTERVAL,
+	exponential: number = DEFAULT_EXP_BACKOFF_EXPONENTIAL,
+	maxRetries: number = DEFAULT_EXP_BACKOFF_RETRIES,
+): Promise<any> {
+	let lastError;
+	for (let i = 0; i < maxRetries; i++) {
+		try {
+			return await fn();
+		} catch (error) {
+			lastError = error;
+			const status = Number(error?.httpCode);
+			if (isStatusCodeRetryable(status)) {
+				//Generate a new sleep time based from interval * exponential^i function
+				const sleepTimeSecs = interval * Math.pow(exponential, i);
+				const sleepTimeMs = sleepTimeSecs * 1000;
+
+				await sleep(sleepTimeMs);
+
+				continue;
+			}
+			throw error;
+		}
+	}
+	//In case all of the calls failed with no status or isStatusCodeRetryable, throw the last error
+	throw lastError;
+}
+
 export async function apiRequestAllItems(
 	this: IHookFunctions | IExecuteFunctions | ILoadOptionsFunctions,
 	requestOptions: IApiRequestOptions,

nodes/Apify/resources/key-value-stores/get-key-value-store-record/execute.ts

@@ -6,6 +6,7 @@ import {
 	NodeOperationError,
 } from 'n8n-workflow';
 import { consts } from '../../../helpers';
+import { retryWithExponentialBackoff } from '../../genericFunctions';
 
 export async function getKeyValueStoreRecord(
 	this: IExecuteFunctions,
@@ -19,15 +20,17 @@ export async function getKeyValueStoreRecord(
 	}
 
 	try {
-		const apiResult = await this.helpers.httpRequestWithAuthentication.call(this, 'apifyApi', {
-			method: 'GET' as IHttpRequestMethods,
-			url: `${consts.APIFY_API_URL}/v2/key-value-stores/${storeId.value}/records/${recordKey.value}`,
-			headers: {
-				'x-apify-integration-platform': 'n8n',
-			},
-			returnFullResponse: true,
-			encoding: 'arraybuffer',
-		});
+		const apiCallFn = () =>
+			this.helpers.httpRequestWithAuthentication.call(this, 'apifyApi', {
+				method: 'GET' as IHttpRequestMethods,
+				url: `${consts.APIFY_API_URL}/v2/key-value-stores/${storeId.value}/records/${recordKey.value}`,
+				headers: {
+					'x-apify-integration-platform': 'n8n',
+				},
+				returnFullResponse: true,
+				encoding: 'arraybuffer',
+			});
+		const apiResult = await retryWithExponentialBackoff(apiCallFn);
 
 		if (!apiResult) {
 			return { json: {} };