Skip to content

fix: Use appropriate timeouts #704

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jun 11, 2025
Merged

fix: Use appropriate timeouts #704

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4d9d077
fix: Use appropriate timeouts
janbuchar Jun 9, 2025
95610ac
Remove accidental BC break
janbuchar Jun 10, 2025
5566109
Reconcile timeout units
janbuchar Jun 10, 2025
59c698d
Fix operator precedence error
janbuchar Jun 10, 2025
4aa39a9
Use milliseconds internally to simplify code
janbuchar Jun 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/apify_client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ import { WebhookDispatchClient } from './resource_clients/webhook_dispatch';
import { WebhookDispatchCollectionClient } from './resource_clients/webhook_dispatch_collection';
import { Statistics } from './statistics';

const DEFAULT_TIMEOUT_SECS = 360;

/**
* ApifyClient is the official library to access [Apify API](https://docs.apify.com/api/v2) from your
* JavaScript applications. It runs both in Node.js and browser.
Expand Down Expand Up @@ -67,7 +69,7 @@ export class ApifyClient {
maxRetries = 8,
minDelayBetweenRetriesMillis = 500,
requestInterceptors = [],
timeoutSecs = 360,
timeoutSecs = DEFAULT_TIMEOUT_SECS,
token,
} = options;

Expand Down
13 changes: 10 additions & 3 deletions src/base/resource_client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,21 @@ import { ApiClient } from './api_client';
*/
const MAX_WAIT_FOR_FINISH = 999999;

export const SMALL_TIMEOUT_SECS = 5; // For fast and common actions. Suitable for idempotent actions.
export const MEDIUM_TIMEOUT_SECS = 30; // For actions that may take longer.
export const DEFAULT_TIMEOUT_SECS = 360; // 6 minutes

/**
* Resource client.
* @private
*/
export class ResourceClient extends ApiClient {
protected async _get<T, R>(options: T = {} as T): Promise<R | undefined> {
protected async _get<T, R>(options: T = {} as T, timeoutSecs?: number): Promise<R | undefined> {
const requestOpts: ApifyRequestConfig = {
url: this._url(),
method: 'GET',
params: this._params(options),
timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : undefined,
};
try {
const response = await this.httpClient.call(requestOpts);
Expand All @@ -34,22 +39,24 @@ export class ResourceClient extends ApiClient {
return undefined;
}

protected async _update<T, R>(newFields: T): Promise<R> {
protected async _update<T, R>(newFields: T, timeoutSecs?: number): Promise<R> {
const response = await this.httpClient.call({
url: this._url(),
method: 'PUT',
params: this._params(),
data: newFields,
timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : undefined,
});
return parseDateFields(pluckData(response.data)) as R;
}

protected async _delete(): Promise<void> {
protected async _delete(timeoutSecs?: number): Promise<void> {
try {
await this.httpClient.call({
url: this._url(),
method: 'DELETE',
params: this._params(),
timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : undefined,
});
} catch (err) {
catchNotFoundOrThrow(err as ApifyApiError);
Expand Down
8 changes: 8 additions & 0 deletions src/http_client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,13 +152,21 @@ export class HttpClient {
this.stats.requests++;
let response: ApifyResponse;
const requestIsStream = isStream(config.data);

try {
if (requestIsStream) {
// Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
// when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
// see also axios/axios#1045
config = { ...config, maxRedirects: 0 };
}

// Increase timeout with each attempt. Max timeout is bounded by the client timeout.
config.timeout = Math.min(
this.timeoutMillis,
(config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1),
);

response = await this.axios.request(config);
if (this._isStatusOk(response.status)) return response;
} catch (err) {
Expand Down
12 changes: 8 additions & 4 deletions src/resource_clients/dataset.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';

import type { ApifyApiError } from '../apify_api_error';
import type { ApiClientSubResourceOptions } from '../base/api_client';
import { ResourceClient } from '../base/resource_client';
import { DEFAULT_TIMEOUT_SECS, MEDIUM_TIMEOUT_SECS, ResourceClient, SMALL_TIMEOUT_SECS } from '../base/resource_client';
import type { ApifyRequestConfig, ApifyResponse } from '../http_client';
import type { PaginatedList } from '../utils';
import { cast, catchNotFoundOrThrow, pluckData } from '../utils';
Expand All @@ -26,7 +26,7 @@ export class DatasetClient<
* https://docs.apify.com/api/v2#/reference/datasets/dataset/get-dataset
*/
async get(): Promise<Dataset | undefined> {
return this._get();
return this._get({}, SMALL_TIMEOUT_SECS);
}

/**
Expand All @@ -35,14 +35,14 @@ export class DatasetClient<
async update(newFields: DatasetClientUpdateOptions): Promise<Dataset> {
ow(newFields, ow.object);

return this._update(newFields);
return this._update(newFields, SMALL_TIMEOUT_SECS);
}

/**
* https://docs.apify.com/api/v2#/reference/datasets/dataset/delete-dataset
*/
async delete(): Promise<void> {
return this._delete();
return this._delete(SMALL_TIMEOUT_SECS);
}

/**
Expand Down Expand Up @@ -70,6 +70,7 @@ export class DatasetClient<
url: this._url('items'),
method: 'GET',
params: this._params(options),
timeout: DEFAULT_TIMEOUT_SECS * 1000,
});

return this._createPaginationList(response, options.desc ?? false);
Expand Down Expand Up @@ -113,6 +114,7 @@ export class DatasetClient<
...options,
}),
forceBuffer: true,
timeout: DEFAULT_TIMEOUT_SECS * 1000,
});

return cast(data);
Expand All @@ -133,6 +135,7 @@ export class DatasetClient<
data: items,
params: this._params(),
doNotRetryTimeouts: true, // see timeout handling in http-client
timeout: MEDIUM_TIMEOUT_SECS * 1000,
});
}

Expand All @@ -144,6 +147,7 @@ export class DatasetClient<
url: this._url('statistics'),
method: 'GET',
params: this._params(),
timeout: SMALL_TIMEOUT_SECS * 1000,
};
try {
const response = await this.httpClient.call(requestOpts);
Expand Down
16 changes: 8 additions & 8 deletions src/resource_clients/key_value_store.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import log from '@apify/log';

import type { ApifyApiError } from '../apify_api_error';
import type { ApiClientSubResourceOptions } from '../base/api_client';
import { ResourceClient } from '../base/resource_client';
import { DEFAULT_TIMEOUT_SECS, MEDIUM_TIMEOUT_SECS, ResourceClient, SMALL_TIMEOUT_SECS } from '../base/resource_client';
import type { ApifyRequestConfig } from '../http_client';
import { cast, catchNotFoundOrThrow, isBuffer, isNode, isStream, parseDateFields, pluckData } from '../utils';

Expand All @@ -27,7 +27,7 @@ export class KeyValueStoreClient extends ResourceClient {
* https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/get-store
*/
async get(): Promise<KeyValueStore | undefined> {
return this._get();
return this._get({}, SMALL_TIMEOUT_SECS);
}

/**
Expand All @@ -36,14 +36,14 @@ export class KeyValueStoreClient extends ResourceClient {
async update(newFields: KeyValueClientUpdateOptions): Promise<KeyValueStore> {
ow(newFields, ow.object);

return this._update(newFields);
return this._update(newFields, DEFAULT_TIMEOUT_SECS);
}

/**
* https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/delete-store
*/
async delete(): Promise<void> {
return this._delete();
return this._delete(SMALL_TIMEOUT_SECS);
}

/**
Expand All @@ -64,6 +64,7 @@ export class KeyValueStoreClient extends ResourceClient {
url: this._url('keys'),
method: 'GET',
params: this._params(options),
timeout: MEDIUM_TIMEOUT_SECS * 1000,
});

return cast(parseDateFields(pluckData(response.data)));
Expand Down Expand Up @@ -138,6 +139,7 @@ export class KeyValueStoreClient extends ResourceClient {
url: this._url(`records/${key}`),
method: 'GET',
params: this._params(),
timeout: DEFAULT_TIMEOUT_SECS * 1000,
};

if (options.buffer) requestOpts.forceBuffer = true;
Expand Down Expand Up @@ -215,12 +217,9 @@ export class KeyValueStoreClient extends ResourceClient {
data: value,
headers: contentType ? { 'content-type': contentType } : undefined,
doNotRetryTimeouts,
timeout: (timeoutSecs ?? DEFAULT_TIMEOUT_SECS) * 1000,
};

if (timeoutSecs != null) {
uploadOpts.timeout = timeoutSecs * 1000;
}

await this.httpClient.call(uploadOpts);
}

Expand All @@ -234,6 +233,7 @@ export class KeyValueStoreClient extends ResourceClient {
url: this._url(`records/${key}`),
method: 'DELETE',
params: this._params(),
timeout: SMALL_TIMEOUT_SECS * 1000,
});
}
}
Expand Down
32 changes: 16 additions & 16 deletions src/resource_clients/request_queue.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import log from '@apify/log';

import type { ApifyApiError } from '../apify_api_error';
import type { ApiClientSubResourceOptions } from '../base/api_client';
import { ResourceClient } from '../base/resource_client';
import { MEDIUM_TIMEOUT_SECS, ResourceClient, SMALL_TIMEOUT_SECS } from '../base/resource_client';
import type { ApifyRequestConfig } from '../http_client';
import {
cast,
Expand Down Expand Up @@ -46,7 +46,7 @@ export class RequestQueueClient extends ResourceClient {
* https://docs.apify.com/api/v2#/reference/request-queues/queue/get-request-queue
*/
async get(): Promise<RequestQueue | undefined> {
return this._get();
return this._get({}, SMALL_TIMEOUT_SECS);
}

/**
Expand All @@ -55,14 +55,14 @@ export class RequestQueueClient extends ResourceClient {
async update(newFields: RequestQueueClientUpdateOptions): Promise<RequestQueue> {
ow(newFields, ow.object);

return this._update(newFields);
return this._update(newFields, SMALL_TIMEOUT_SECS);
}

/**
* https://docs.apify.com/api/v2#/reference/request-queues/queue/delete-request-queue
*/
async delete(): Promise<void> {
return this._delete();
return this._delete(SMALL_TIMEOUT_SECS);
}

/**
Expand All @@ -79,7 +79,7 @@ export class RequestQueueClient extends ResourceClient {
const response = await this.httpClient.call({
url: this._url('head'),
method: 'GET',
timeout: this.timeoutMillis,
timeout: Math.min(SMALL_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params({
limit: options.limit,
clientKey: this.clientKey,
Expand All @@ -106,7 +106,7 @@ export class RequestQueueClient extends ResourceClient {
const response = await this.httpClient.call({
url: this._url('head/lock'),
method: 'POST',
timeout: this.timeoutMillis,
timeout: Math.min(MEDIUM_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params({
limit: options.limit,
lockSecs: options.lockSecs,
Expand Down Expand Up @@ -141,7 +141,7 @@ export class RequestQueueClient extends ResourceClient {
const response = await this.httpClient.call({
url: this._url('requests'),
method: 'POST',
timeout: this.timeoutMillis,
timeout: Math.min(SMALL_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
data: request,
params: this._params({
forefront: options.forefront,
Expand Down Expand Up @@ -182,7 +182,7 @@ export class RequestQueueClient extends ResourceClient {
const { data } = await this.httpClient.call({
url: this._url('requests/batch'),
method: 'POST',
timeout: this.timeoutMillis,
timeout: Math.min(MEDIUM_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
data: requests,
params: this._params({
forefront: options.forefront,
Expand Down Expand Up @@ -343,7 +343,7 @@ export class RequestQueueClient extends ResourceClient {
const { data } = await this.httpClient.call({
url: this._url('requests/batch'),
method: 'DELETE',
timeout: this.timeoutMillis,
timeout: Math.min(SMALL_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
data: requests,
params: this._params({
clientKey: this.clientKey,
Expand All @@ -361,7 +361,7 @@ export class RequestQueueClient extends ResourceClient {
const requestOpts: ApifyRequestConfig = {
url: this._url(`requests/${id}`),
method: 'GET',
timeout: this.timeoutMillis,
timeout: Math.min(SMALL_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params(),
};
try {
Expand Down Expand Up @@ -398,7 +398,7 @@ export class RequestQueueClient extends ResourceClient {
const response = await this.httpClient.call({
url: this._url(`requests/${request.id}`),
method: 'PUT',
timeout: this.timeoutMillis,
timeout: Math.min(MEDIUM_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
data: request,
params: this._params({
forefront: options.forefront,
Expand All @@ -415,7 +415,7 @@ export class RequestQueueClient extends ResourceClient {
await this.httpClient.call({
url: this._url(`requests/${id}`),
method: 'DELETE',
timeout: this.timeoutMillis,
timeout: Math.min(SMALL_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params({
clientKey: this.clientKey,
}),
Expand All @@ -441,7 +441,7 @@ export class RequestQueueClient extends ResourceClient {
const response = await this.httpClient.call({
url: this._url(`requests/${id}/lock`),
method: 'PUT',
timeout: this.timeoutMillis,
timeout: Math.min(MEDIUM_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params({
forefront: options.forefront,
lockSecs: options.lockSecs,
Expand All @@ -467,7 +467,7 @@ export class RequestQueueClient extends ResourceClient {
await this.httpClient.call({
url: this._url(`requests/${id}/lock`),
method: 'DELETE',
timeout: this.timeoutMillis,
timeout: Math.min(SMALL_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params({
forefront: options.forefront,
clientKey: this.clientKey,
Expand All @@ -492,7 +492,7 @@ export class RequestQueueClient extends ResourceClient {
const response = await this.httpClient.call({
url: this._url('requests'),
method: 'GET',
timeout: this.timeoutMillis,
timeout: Math.min(MEDIUM_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params({
limit: options.limit,
exclusiveStartId: options.exclusiveStartId,
Expand All @@ -510,7 +510,7 @@ export class RequestQueueClient extends ResourceClient {
const response = await this.httpClient.call({
url: this._url('requests/unlock'),
method: 'POST',
timeout: this.timeoutMillis,
timeout: Math.min(MEDIUM_TIMEOUT_SECS * 1000, this.timeoutMillis ?? Infinity),
params: this._params({
clientKey: this.clientKey,
}),
Expand Down
Loading