diff --git a/src/apify_client.ts b/src/apify_client.ts index 0cf6d202..4ff26593 100644 --- a/src/apify_client.ts +++ b/src/apify_client.ts @@ -33,6 +33,8 @@ import { WebhookDispatchClient } from './resource_clients/webhook_dispatch'; import { WebhookDispatchCollectionClient } from './resource_clients/webhook_dispatch_collection'; import { Statistics } from './statistics'; +const DEFAULT_TIMEOUT_SECS = 360; + /** * ApifyClient is the official library to access [Apify API](https://docs.apify.com/api/v2) from your * JavaScript applications. It runs both in Node.js and browser. @@ -67,7 +69,7 @@ export class ApifyClient { maxRetries = 8, minDelayBetweenRetriesMillis = 500, requestInterceptors = [], - timeoutSecs = 360, + timeoutSecs = DEFAULT_TIMEOUT_SECS, token, } = options; diff --git a/src/base/resource_client.ts b/src/base/resource_client.ts index e87bfac9..0a57bd3f 100644 --- a/src/base/resource_client.ts +++ b/src/base/resource_client.ts @@ -13,16 +13,21 @@ import { ApiClient } from './api_client'; */ const MAX_WAIT_FOR_FINISH = 999999; +export const SMALL_TIMEOUT_MILLIS = 5 * 1000; // For fast and common actions. Suitable for idempotent actions. +export const MEDIUM_TIMEOUT_MILLIS = 30 * 1000; // For actions that may take longer. +export const DEFAULT_TIMEOUT_MILLIS = 360 * 1000; // 6 minutes + /** * Resource client. * @private */ export class ResourceClient extends ApiClient { - protected async _get(options: T = {} as T): Promise { + protected async _get(options: T = {} as T, timeoutMillis?: number): Promise { const requestOpts: ApifyRequestConfig = { url: this._url(), method: 'GET', params: this._params(options), + timeout: timeoutMillis, }; try { const response = await this.httpClient.call(requestOpts); @@ -34,22 +39,24 @@ export class ResourceClient extends ApiClient { return undefined; } - protected async _update(newFields: T): Promise { + protected async _update(newFields: T, timeoutMillis?: number): Promise { const response = await this.httpClient.call({ url: this._url(), method: 'PUT', params: this._params(), data: newFields, + timeout: timeoutMillis, }); return parseDateFields(pluckData(response.data)) as R; } - protected async _delete(): Promise { + protected async _delete(timeoutMillis?: number): Promise { try { await this.httpClient.call({ url: this._url(), method: 'DELETE', params: this._params(), + timeout: timeoutMillis, }); } catch (err) { catchNotFoundOrThrow(err as ApifyApiError); diff --git a/src/http_client.ts b/src/http_client.ts index 6821425c..46009cac 100644 --- a/src/http_client.ts +++ b/src/http_client.ts @@ -152,6 +152,7 @@ export class HttpClient { this.stats.requests++; let response: ApifyResponse; const requestIsStream = isStream(config.data); + try { if (requestIsStream) { // Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered @@ -159,6 +160,13 @@ export class HttpClient { // see also axios/axios#1045 config = { ...config, maxRedirects: 0 }; } + + // Increase timeout with each attempt. Max timeout is bounded by the client timeout. + config.timeout = Math.min( + this.timeoutMillis, + (config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1), + ); + response = await this.axios.request(config); if (this._isStatusOk(response.status)) return response; } catch (err) { diff --git a/src/resource_clients/dataset.ts b/src/resource_clients/dataset.ts index 466756ca..4929102b 100644 --- a/src/resource_clients/dataset.ts +++ b/src/resource_clients/dataset.ts @@ -4,7 +4,12 @@ import type { STORAGE_GENERAL_ACCESS } from '@apify/consts'; import type { ApifyApiError } from '../apify_api_error'; import type { ApiClientSubResourceOptions } from '../base/api_client'; -import { ResourceClient } from '../base/resource_client'; +import { + DEFAULT_TIMEOUT_MILLIS, + MEDIUM_TIMEOUT_MILLIS, + ResourceClient, + SMALL_TIMEOUT_MILLIS, +} from '../base/resource_client'; import type { ApifyRequestConfig, ApifyResponse } from '../http_client'; import type { PaginatedList } from '../utils'; import { cast, catchNotFoundOrThrow, pluckData } from '../utils'; @@ -26,7 +31,7 @@ export class DatasetClient< * https://docs.apify.com/api/v2#/reference/datasets/dataset/get-dataset */ async get(): Promise { - return this._get(); + return this._get({}, SMALL_TIMEOUT_MILLIS); } /** @@ -35,14 +40,14 @@ export class DatasetClient< async update(newFields: DatasetClientUpdateOptions): Promise { ow(newFields, ow.object); - return this._update(newFields); + return this._update(newFields, SMALL_TIMEOUT_MILLIS); } /** * https://docs.apify.com/api/v2#/reference/datasets/dataset/delete-dataset */ async delete(): Promise { - return this._delete(); + return this._delete(SMALL_TIMEOUT_MILLIS); } /** @@ -70,6 +75,7 @@ export class DatasetClient< url: this._url('items'), method: 'GET', params: this._params(options), + timeout: DEFAULT_TIMEOUT_MILLIS, }); return this._createPaginationList(response, options.desc ?? false); @@ -113,6 +119,7 @@ export class DatasetClient< ...options, }), forceBuffer: true, + timeout: DEFAULT_TIMEOUT_MILLIS, }); return cast(data); @@ -133,6 +140,7 @@ export class DatasetClient< data: items, params: this._params(), doNotRetryTimeouts: true, // see timeout handling in http-client + timeout: MEDIUM_TIMEOUT_MILLIS, }); } @@ -144,6 +152,7 @@ export class DatasetClient< url: this._url('statistics'), method: 'GET', params: this._params(), + timeout: SMALL_TIMEOUT_MILLIS, }; try { const response = await this.httpClient.call(requestOpts); diff --git a/src/resource_clients/key_value_store.ts b/src/resource_clients/key_value_store.ts index 23bb1ba5..01794e86 100644 --- a/src/resource_clients/key_value_store.ts +++ b/src/resource_clients/key_value_store.ts @@ -8,7 +8,12 @@ import log from '@apify/log'; import type { ApifyApiError } from '../apify_api_error'; import type { ApiClientSubResourceOptions } from '../base/api_client'; -import { ResourceClient } from '../base/resource_client'; +import { + DEFAULT_TIMEOUT_MILLIS, + MEDIUM_TIMEOUT_MILLIS, + ResourceClient, + SMALL_TIMEOUT_MILLIS, +} from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, catchNotFoundOrThrow, isBuffer, isNode, isStream, parseDateFields, pluckData } from '../utils'; @@ -27,7 +32,7 @@ export class KeyValueStoreClient extends ResourceClient { * https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/get-store */ async get(): Promise { - return this._get(); + return this._get({}, SMALL_TIMEOUT_MILLIS); } /** @@ -36,14 +41,14 @@ export class KeyValueStoreClient extends ResourceClient { async update(newFields: KeyValueClientUpdateOptions): Promise { ow(newFields, ow.object); - return this._update(newFields); + return this._update(newFields, DEFAULT_TIMEOUT_MILLIS); } /** * https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/delete-store */ async delete(): Promise { - return this._delete(); + return this._delete(SMALL_TIMEOUT_MILLIS); } /** @@ -64,6 +69,7 @@ export class KeyValueStoreClient extends ResourceClient { url: this._url('keys'), method: 'GET', params: this._params(options), + timeout: MEDIUM_TIMEOUT_MILLIS, }); return cast(parseDateFields(pluckData(response.data))); @@ -138,6 +144,7 @@ export class KeyValueStoreClient extends ResourceClient { url: this._url(`records/${key}`), method: 'GET', params: this._params(), + timeout: DEFAULT_TIMEOUT_MILLIS, }; if (options.buffer) requestOpts.forceBuffer = true; @@ -215,12 +222,9 @@ export class KeyValueStoreClient extends ResourceClient { data: value, headers: contentType ? { 'content-type': contentType } : undefined, doNotRetryTimeouts, + timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : DEFAULT_TIMEOUT_MILLIS, }; - if (timeoutSecs != null) { - uploadOpts.timeout = timeoutSecs * 1000; - } - await this.httpClient.call(uploadOpts); } @@ -234,6 +238,7 @@ export class KeyValueStoreClient extends ResourceClient { url: this._url(`records/${key}`), method: 'DELETE', params: this._params(), + timeout: SMALL_TIMEOUT_MILLIS, }); } } diff --git a/src/resource_clients/request_queue.ts b/src/resource_clients/request_queue.ts index 8adfeb49..7cec8771 100644 --- a/src/resource_clients/request_queue.ts +++ b/src/resource_clients/request_queue.ts @@ -7,7 +7,7 @@ import log from '@apify/log'; import type { ApifyApiError } from '../apify_api_error'; import type { ApiClientSubResourceOptions } from '../base/api_client'; -import { ResourceClient } from '../base/resource_client'; +import { MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS } from '../base/resource_client'; import type { ApifyRequestConfig } from '../http_client'; import { cast, @@ -46,7 +46,7 @@ export class RequestQueueClient extends ResourceClient { * https://docs.apify.com/api/v2#/reference/request-queues/queue/get-request-queue */ async get(): Promise { - return this._get(); + return this._get({}, SMALL_TIMEOUT_MILLIS); } /** @@ -55,14 +55,14 @@ export class RequestQueueClient extends ResourceClient { async update(newFields: RequestQueueClientUpdateOptions): Promise { ow(newFields, ow.object); - return this._update(newFields); + return this._update(newFields, SMALL_TIMEOUT_MILLIS); } /** * https://docs.apify.com/api/v2#/reference/request-queues/queue/delete-request-queue */ async delete(): Promise { - return this._delete(); + return this._delete(SMALL_TIMEOUT_MILLIS); } /** @@ -79,7 +79,7 @@ export class RequestQueueClient extends ResourceClient { const response = await this.httpClient.call({ url: this._url('head'), method: 'GET', - timeout: this.timeoutMillis, + timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params({ limit: options.limit, clientKey: this.clientKey, @@ -106,7 +106,7 @@ export class RequestQueueClient extends ResourceClient { const response = await this.httpClient.call({ url: this._url('head/lock'), method: 'POST', - timeout: this.timeoutMillis, + timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params({ limit: options.limit, lockSecs: options.lockSecs, @@ -141,7 +141,7 @@ export class RequestQueueClient extends ResourceClient { const response = await this.httpClient.call({ url: this._url('requests'), method: 'POST', - timeout: this.timeoutMillis, + timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), data: request, params: this._params({ forefront: options.forefront, @@ -182,7 +182,7 @@ export class RequestQueueClient extends ResourceClient { const { data } = await this.httpClient.call({ url: this._url('requests/batch'), method: 'POST', - timeout: this.timeoutMillis, + timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), data: requests, params: this._params({ forefront: options.forefront, @@ -343,7 +343,7 @@ export class RequestQueueClient extends ResourceClient { const { data } = await this.httpClient.call({ url: this._url('requests/batch'), method: 'DELETE', - timeout: this.timeoutMillis, + timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), data: requests, params: this._params({ clientKey: this.clientKey, @@ -361,7 +361,7 @@ export class RequestQueueClient extends ResourceClient { const requestOpts: ApifyRequestConfig = { url: this._url(`requests/${id}`), method: 'GET', - timeout: this.timeoutMillis, + timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params(), }; try { @@ -398,7 +398,7 @@ export class RequestQueueClient extends ResourceClient { const response = await this.httpClient.call({ url: this._url(`requests/${request.id}`), method: 'PUT', - timeout: this.timeoutMillis, + timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), data: request, params: this._params({ forefront: options.forefront, @@ -415,7 +415,7 @@ export class RequestQueueClient extends ResourceClient { await this.httpClient.call({ url: this._url(`requests/${id}`), method: 'DELETE', - timeout: this.timeoutMillis, + timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params({ clientKey: this.clientKey, }), @@ -441,7 +441,7 @@ export class RequestQueueClient extends ResourceClient { const response = await this.httpClient.call({ url: this._url(`requests/${id}/lock`), method: 'PUT', - timeout: this.timeoutMillis, + timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params({ forefront: options.forefront, lockSecs: options.lockSecs, @@ -467,7 +467,7 @@ export class RequestQueueClient extends ResourceClient { await this.httpClient.call({ url: this._url(`requests/${id}/lock`), method: 'DELETE', - timeout: this.timeoutMillis, + timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params({ forefront: options.forefront, clientKey: this.clientKey, @@ -492,7 +492,7 @@ export class RequestQueueClient extends ResourceClient { const response = await this.httpClient.call({ url: this._url('requests'), method: 'GET', - timeout: this.timeoutMillis, + timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params({ limit: options.limit, exclusiveStartId: options.exclusiveStartId, @@ -510,7 +510,7 @@ export class RequestQueueClient extends ResourceClient { const response = await this.httpClient.call({ url: this._url('requests/unlock'), method: 'POST', - timeout: this.timeoutMillis, + timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity), params: this._params({ clientKey: this.clientKey, }), diff --git a/test/client_timeouts.test.js b/test/client_timeouts.test.js new file mode 100644 index 00000000..80dbf24c --- /dev/null +++ b/test/client_timeouts.test.js @@ -0,0 +1,142 @@ +const { ApifyClient } = require('../dist/index.js'); + +// Mock for testing timeout functionality +class MockHttpClient { + constructor(timeoutSecs) { + this.timeoutSecs = timeoutSecs * 1000; // Convert to milliseconds + this.callHistory = []; + this.stats = { + addRateLimitError: jest.fn(), + }; + } + + async call(config) { + // Track the call for assertions + this.callHistory.push({ + ...config, + timeoutUsed: config.timeout, + }); + + // Simulate successful response + return { + data: { data: { id: 'test-id' } }, + status: 200, + headers: {}, + }; + } +} + +// Test parameters: [ClientClass, methodName, expectedTimeoutMillis, methodArgs] +const timeoutTestParams = [ + // Dataset Client + ['DatasetClient', 'get', 5000, []], + ['DatasetClient', 'update', 5000, [{ name: 'new-name' }]], + ['DatasetClient', 'delete', 5000, []], + ['DatasetClient', 'listItems', 360000, []], + ['DatasetClient', 'downloadItems', 360000, ['json']], + ['DatasetClient', 'pushItems', 30000, [[{ test: 'data' }]]], + ['DatasetClient', 'getStatistics', 5000, []], + + // KeyValueStore Client + ['KeyValueStoreClient', 'get', 5000, []], + ['KeyValueStoreClient', 'update', 360000, [{}]], + ['KeyValueStoreClient', 'delete', 5000, []], + ['KeyValueStoreClient', 'listKeys', 30000, []], + ['KeyValueStoreClient', 'getRecord', 360000, ['test-key']], + ['KeyValueStoreClient', 'setRecord', 360000, [{ key: 'test-key', value: 'some-value' }]], + ['KeyValueStoreClient', 'deleteRecord', 5000, ['test-key']], + + // RequestQueue Client + ['RequestQueueClient', 'get', 5000, []], + ['RequestQueueClient', 'update', 5000, [{ name: 'new-name' }]], + ['RequestQueueClient', 'delete', 5000, []], + ['RequestQueueClient', 'listHead', 5000, []], + ['RequestQueueClient', 'listAndLockHead', 30000, [{ lockSecs: 10 }]], + ['RequestQueueClient', 'addRequest', 5000, [{ url: 'https://example.com', uniqueKey: 'test' }]], + ['RequestQueueClient', 'getRequest', 5000, ['request-id']], + [ + 'RequestQueueClient', + 'updateRequest', + 30000, + [{ id: 'request-id', url: 'https://example.com', uniqueKey: 'test' }], + ], + ['RequestQueueClient', 'deleteRequest', 5000, ['request-id']], + ['RequestQueueClient', 'prolongRequestLock', 30000, ['request-id', { lockSecs: 10 }]], + ['RequestQueueClient', 'deleteRequestLock', 5000, ['request-id']], + ['RequestQueueClient', 'batchAddRequests', 30000, [[{ uniqueKey: 'request-key' }], {}]], + ['RequestQueueClient', 'batchDeleteRequests', 5000, [[{ id: 'request-id' }]]], + ['RequestQueueClient', 'listRequests', 30000, []], +]; + +describe('Client Timeout Tests', () => { + let client; + let mockHttpClient; + + beforeEach(() => { + mockHttpClient = new MockHttpClient(360); // 6 minutes default + client = new ApifyClient(); + // Replace the http client with our mock + client.httpClient = mockHttpClient; + }); + + describe('Dynamic Timeout with Exponential Backoff', () => { + test('timeout increases with each retry attempt', () => { + const initialTimeoutSecs = 5; + const clientTimeoutSecs = 60; + + // Attempt 1: 5 seconds + const timeout1 = Math.min(clientTimeoutSecs, initialTimeoutSecs * 2 ** (1 - 1)); + expect(timeout1).toBe(5); + + // Attempt 2: 10 seconds + const timeout2 = Math.min(clientTimeoutSecs, initialTimeoutSecs * 2 ** (2 - 1)); + expect(timeout2).toBe(10); + + // Attempt 3: 20 seconds + const timeout3 = Math.min(clientTimeoutSecs, initialTimeoutSecs * 2 ** (3 - 1)); + expect(timeout3).toBe(20); + + // Attempt 4: 40 seconds + const timeout4 = Math.min(clientTimeoutSecs, initialTimeoutSecs * 2 ** (4 - 1)); + expect(timeout4).toBe(40); + + // Attempt 5: Would be 80 seconds, but limited to client timeout (60) + const timeout5 = Math.min(clientTimeoutSecs, initialTimeoutSecs * 2 ** (5 - 1)); + expect(timeout5).toBe(60); + }); + }); + + describe.each(timeoutTestParams)( + 'Specific timeouts for specific endpoints', + (clientType, methodName, expectedTimeoutMillis, methodArgs) => { + test(`${clientType}.${methodName}() uses ${expectedTimeoutMillis} millisecond timeout`, async () => { + let clientInstance; + + // Create the appropriate client instance + switch (clientType) { + case 'DatasetClient': + clientInstance = client.dataset('test-id'); + break; + case 'KeyValueStoreClient': + clientInstance = client.keyValueStore('test-id'); + break; + case 'RequestQueueClient': + clientInstance = client.requestQueue('test-id'); + break; + default: + throw new Error(`Unknown client type: ${clientType}`); + } + + // Replace with our mock + clientInstance.httpClient = mockHttpClient; + + // Call the method + await clientInstance[methodName](...methodArgs); + + // Check the timeout was set correctly + const lastCall = mockHttpClient.callHistory[mockHttpClient.callHistory.length - 1]; + expect(lastCall.timeout).toBe(expectedTimeoutMillis); + }); + }, + ); +});