Skip to content

Commit b896bf2

Browse files
janbucharjanbuchar
authored
fix: Use appropriate timeouts (#704)
- closes #685 - modelled after apify/apify-client-python#384
1 parent 9c93cbb commit b896bf2

File tree

7 files changed

+205
-32
lines changed

7 files changed

+205
-32
lines changed

‎src/apify_client.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ import { WebhookDispatchClient } from './resource_clients/webhook_dispatch';
3333
import { WebhookDispatchCollectionClient } from './resource_clients/webhook_dispatch_collection';
3434
import { Statistics } from './statistics';
3535

36+
const DEFAULT_TIMEOUT_SECS = 360;
37+
3638
/**
3739
* ApifyClient is the official library to access [Apify API](https://docs.apify.com/api/v2) from your
3840
* JavaScript applications. It runs both in Node.js and browser.
@@ -67,7 +69,7 @@ export class ApifyClient {
6769
maxRetries = 8,
6870
minDelayBetweenRetriesMillis = 500,
6971
requestInterceptors = [],
70-
timeoutSecs = 360,
72+
timeoutSecs = DEFAULT_TIMEOUT_SECS,
7173
token,
7274
} = options;
7375

‎src/base/resource_client.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,21 @@ import { ApiClient } from './api_client';
1313
*/
1414
const MAX_WAIT_FOR_FINISH = 999999;
1515

16+
export const SMALL_TIMEOUT_MILLIS = 5 * 1000; // For fast and common actions. Suitable for idempotent actions.
17+
export const MEDIUM_TIMEOUT_MILLIS = 30 * 1000; // For actions that may take longer.
18+
export const DEFAULT_TIMEOUT_MILLIS = 360 * 1000; // 6 minutes
19+
1620
/**
1721
* Resource client.
1822
* @private
1923
*/
2024
export class ResourceClient extends ApiClient {
21-
protected async _get<T, R>(options: T = {} as T): Promise<R | undefined> {
25+
protected async _get<T, R>(options: T = {} as T, timeoutMillis?: number): Promise<R | undefined> {
2226
const requestOpts: ApifyRequestConfig = {
2327
url: this._url(),
2428
method: 'GET',
2529
params: this._params(options),
30+
timeout: timeoutMillis,
2631
};
2732
try {
2833
const response = await this.httpClient.call(requestOpts);
@@ -34,22 +39,24 @@ export class ResourceClient extends ApiClient {
3439
return undefined;
3540
}
3641

37-
protected async _update<T, R>(newFields: T): Promise<R> {
42+
protected async _update<T, R>(newFields: T, timeoutMillis?: number): Promise<R> {
3843
const response = await this.httpClient.call({
3944
url: this._url(),
4045
method: 'PUT',
4146
params: this._params(),
4247
data: newFields,
48+
timeout: timeoutMillis,
4349
});
4450
return parseDateFields(pluckData(response.data)) as R;
4551
}
4652

47-
protected async _delete(): Promise<void> {
53+
protected async _delete(timeoutMillis?: number): Promise<void> {
4854
try {
4955
await this.httpClient.call({
5056
url: this._url(),
5157
method: 'DELETE',
5258
params: this._params(),
59+
timeout: timeoutMillis,
5360
});
5461
} catch (err) {
5562
catchNotFoundOrThrow(err as ApifyApiError);

‎src/http_client.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,13 +152,21 @@ export class HttpClient {
152152
this.stats.requests++;
153153
let response: ApifyResponse;
154154
const requestIsStream = isStream(config.data);
155+
155156
try {
156157
if (requestIsStream) {
157158
// Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
158159
// when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
159160
// see also axios/axios#1045
160161
config = { ...config, maxRedirects: 0 };
161162
}
163+
164+
// Increase timeout with each attempt. Max timeout is bounded by the client timeout.
165+
config.timeout = Math.min(
166+
this.timeoutMillis,
167+
(config.timeout ?? this.timeoutMillis) * 2 ** (attempt - 1),
168+
);
169+
162170
response = await this.axios.request(config);
163171
if (this._isStatusOk(response.status)) return response;
164172
} catch (err) {

‎src/resource_clients/dataset.ts‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,12 @@ import type { STORAGE_GENERAL_ACCESS } from '@apify/consts';
44

55
import type { ApifyApiError } from '../apify_api_error';
66
import type { ApiClientSubResourceOptions } from '../base/api_client';
7-
import { ResourceClient } from '../base/resource_client';
7+
import {
8+
DEFAULT_TIMEOUT_MILLIS,
9+
MEDIUM_TIMEOUT_MILLIS,
10+
ResourceClient,
11+
SMALL_TIMEOUT_MILLIS,
12+
} from '../base/resource_client';
813
import type { ApifyRequestConfig, ApifyResponse } from '../http_client';
914
import type { PaginatedList } from '../utils';
1015
import { cast, catchNotFoundOrThrow, pluckData } from '../utils';
@@ -26,7 +31,7 @@ export class DatasetClient<
2631
* https://docs.apify.com/api/v2#/reference/datasets/dataset/get-dataset
2732
*/
2833
async get(): Promise<Dataset | undefined> {
29-
return this._get();
34+
return this._get({}, SMALL_TIMEOUT_MILLIS);
3035
}
3136

3237
/**
@@ -35,14 +40,14 @@ export class DatasetClient<
3540
async update(newFields: DatasetClientUpdateOptions): Promise<Dataset> {
3641
ow(newFields, ow.object);
3742

38-
return this._update(newFields);
43+
return this._update(newFields, SMALL_TIMEOUT_MILLIS);
3944
}
4045

4146
/**
4247
* https://docs.apify.com/api/v2#/reference/datasets/dataset/delete-dataset
4348
*/
4449
async delete(): Promise<void> {
45-
return this._delete();
50+
return this._delete(SMALL_TIMEOUT_MILLIS);
4651
}
4752

4853
/**
@@ -70,6 +75,7 @@ export class DatasetClient<
7075
url: this._url('items'),
7176
method: 'GET',
7277
params: this._params(options),
78+
timeout: DEFAULT_TIMEOUT_MILLIS,
7379
});
7480

7581
return this._createPaginationList(response, options.desc ?? false);
@@ -113,6 +119,7 @@ export class DatasetClient<
113119
...options,
114120
}),
115121
forceBuffer: true,
122+
timeout: DEFAULT_TIMEOUT_MILLIS,
116123
});
117124

118125
return cast(data);
@@ -133,6 +140,7 @@ export class DatasetClient<
133140
data: items,
134141
params: this._params(),
135142
doNotRetryTimeouts: true, // see timeout handling in http-client
143+
timeout: MEDIUM_TIMEOUT_MILLIS,
136144
});
137145
}
138146

@@ -144,6 +152,7 @@ export class DatasetClient<
144152
url: this._url('statistics'),
145153
method: 'GET',
146154
params: this._params(),
155+
timeout: SMALL_TIMEOUT_MILLIS,
147156
};
148157
try {
149158
const response = await this.httpClient.call(requestOpts);

‎src/resource_clients/key_value_store.ts‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,12 @@ import log from '@apify/log';
88

99
import type { ApifyApiError } from '../apify_api_error';
1010
import type { ApiClientSubResourceOptions } from '../base/api_client';
11-
import { ResourceClient } from '../base/resource_client';
11+
import {
12+
DEFAULT_TIMEOUT_MILLIS,
13+
MEDIUM_TIMEOUT_MILLIS,
14+
ResourceClient,
15+
SMALL_TIMEOUT_MILLIS,
16+
} from '../base/resource_client';
1217
import type { ApifyRequestConfig } from '../http_client';
1318
import { cast, catchNotFoundOrThrow, isBuffer, isNode, isStream, parseDateFields, pluckData } from '../utils';
1419

@@ -27,7 +32,7 @@ export class KeyValueStoreClient extends ResourceClient {
2732
* https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/get-store
2833
*/
2934
async get(): Promise<KeyValueStore | undefined> {
30-
return this._get();
35+
return this._get({}, SMALL_TIMEOUT_MILLIS);
3136
}
3237

3338
/**
@@ -36,14 +41,14 @@ export class KeyValueStoreClient extends ResourceClient {
3641
async update(newFields: KeyValueClientUpdateOptions): Promise<KeyValueStore> {
3742
ow(newFields, ow.object);
3843

39-
return this._update(newFields);
44+
return this._update(newFields, DEFAULT_TIMEOUT_MILLIS);
4045
}
4146

4247
/**
4348
* https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/delete-store
4449
*/
4550
async delete(): Promise<void> {
46-
return this._delete();
51+
return this._delete(SMALL_TIMEOUT_MILLIS);
4752
}
4853

4954
/**
@@ -64,6 +69,7 @@ export class KeyValueStoreClient extends ResourceClient {
6469
url: this._url('keys'),
6570
method: 'GET',
6671
params: this._params(options),
72+
timeout: MEDIUM_TIMEOUT_MILLIS,
6773
});
6874

6975
return cast(parseDateFields(pluckData(response.data)));
@@ -138,6 +144,7 @@ export class KeyValueStoreClient extends ResourceClient {
138144
url: this._url(`records/${key}`),
139145
method: 'GET',
140146
params: this._params(),
147+
timeout: DEFAULT_TIMEOUT_MILLIS,
141148
};
142149

143150
if (options.buffer) requestOpts.forceBuffer = true;
@@ -215,12 +222,9 @@ export class KeyValueStoreClient extends ResourceClient {
215222
data: value,
216223
headers: contentType ? { 'content-type': contentType } : undefined,
217224
doNotRetryTimeouts,
225+
timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : DEFAULT_TIMEOUT_MILLIS,
218226
};
219227

220-
if (timeoutSecs != null) {
221-
uploadOpts.timeout = timeoutSecs * 1000;
222-
}
223-
224228
await this.httpClient.call(uploadOpts);
225229
}
226230

@@ -234,6 +238,7 @@ export class KeyValueStoreClient extends ResourceClient {
234238
url: this._url(`records/${key}`),
235239
method: 'DELETE',
236240
params: this._params(),
241+
timeout: SMALL_TIMEOUT_MILLIS,
237242
});
238243
}
239244
}

‎src/resource_clients/request_queue.ts‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import log from '@apify/log';
77

88
import type { ApifyApiError } from '../apify_api_error';
99
import type { ApiClientSubResourceOptions } from '../base/api_client';
10-
import { ResourceClient } from '../base/resource_client';
10+
import { MEDIUM_TIMEOUT_MILLIS, ResourceClient, SMALL_TIMEOUT_MILLIS } from '../base/resource_client';
1111
import type { ApifyRequestConfig } from '../http_client';
1212
import {
1313
cast,
@@ -46,7 +46,7 @@ export class RequestQueueClient extends ResourceClient {
4646
* https://docs.apify.com/api/v2#/reference/request-queues/queue/get-request-queue
4747
*/
4848
async get(): Promise<RequestQueue | undefined> {
49-
return this._get();
49+
return this._get({}, SMALL_TIMEOUT_MILLIS);
5050
}
5151

5252
/**
@@ -55,14 +55,14 @@ export class RequestQueueClient extends ResourceClient {
5555
async update(newFields: RequestQueueClientUpdateOptions): Promise<RequestQueue> {
5656
ow(newFields, ow.object);
5757

58-
return this._update(newFields);
58+
return this._update(newFields, SMALL_TIMEOUT_MILLIS);
5959
}
6060

6161
/**
6262
* https://docs.apify.com/api/v2#/reference/request-queues/queue/delete-request-queue
6363
*/
6464
async delete(): Promise<void> {
65-
return this._delete();
65+
return this._delete(SMALL_TIMEOUT_MILLIS);
6666
}
6767

6868
/**
@@ -79,7 +79,7 @@ export class RequestQueueClient extends ResourceClient {
7979
const response = await this.httpClient.call({
8080
url: this._url('head'),
8181
method: 'GET',
82-
timeout: this.timeoutMillis,
82+
timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
8383
params: this._params({
8484
limit: options.limit,
8585
clientKey: this.clientKey,
@@ -106,7 +106,7 @@ export class RequestQueueClient extends ResourceClient {
106106
const response = await this.httpClient.call({
107107
url: this._url('head/lock'),
108108
method: 'POST',
109-
timeout: this.timeoutMillis,
109+
timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
110110
params: this._params({
111111
limit: options.limit,
112112
lockSecs: options.lockSecs,
@@ -141,7 +141,7 @@ export class RequestQueueClient extends ResourceClient {
141141
const response = await this.httpClient.call({
142142
url: this._url('requests'),
143143
method: 'POST',
144-
timeout: this.timeoutMillis,
144+
timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
145145
data: request,
146146
params: this._params({
147147
forefront: options.forefront,
@@ -182,7 +182,7 @@ export class RequestQueueClient extends ResourceClient {
182182
const { data } = await this.httpClient.call({
183183
url: this._url('requests/batch'),
184184
method: 'POST',
185-
timeout: this.timeoutMillis,
185+
timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
186186
data: requests,
187187
params: this._params({
188188
forefront: options.forefront,
@@ -343,7 +343,7 @@ export class RequestQueueClient extends ResourceClient {
343343
const { data } = await this.httpClient.call({
344344
url: this._url('requests/batch'),
345345
method: 'DELETE',
346-
timeout: this.timeoutMillis,
346+
timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
347347
data: requests,
348348
params: this._params({
349349
clientKey: this.clientKey,
@@ -361,7 +361,7 @@ export class RequestQueueClient extends ResourceClient {
361361
const requestOpts: ApifyRequestConfig = {
362362
url: this._url(`requests/${id}`),
363363
method: 'GET',
364-
timeout: this.timeoutMillis,
364+
timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
365365
params: this._params(),
366366
};
367367
try {
@@ -398,7 +398,7 @@ export class RequestQueueClient extends ResourceClient {
398398
const response = await this.httpClient.call({
399399
url: this._url(`requests/${request.id}`),
400400
method: 'PUT',
401-
timeout: this.timeoutMillis,
401+
timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
402402
data: request,
403403
params: this._params({
404404
forefront: options.forefront,
@@ -415,7 +415,7 @@ export class RequestQueueClient extends ResourceClient {
415415
await this.httpClient.call({
416416
url: this._url(`requests/${id}`),
417417
method: 'DELETE',
418-
timeout: this.timeoutMillis,
418+
timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
419419
params: this._params({
420420
clientKey: this.clientKey,
421421
}),
@@ -441,7 +441,7 @@ export class RequestQueueClient extends ResourceClient {
441441
const response = await this.httpClient.call({
442442
url: this._url(`requests/${id}/lock`),
443443
method: 'PUT',
444-
timeout: this.timeoutMillis,
444+
timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
445445
params: this._params({
446446
forefront: options.forefront,
447447
lockSecs: options.lockSecs,
@@ -467,7 +467,7 @@ export class RequestQueueClient extends ResourceClient {
467467
await this.httpClient.call({
468468
url: this._url(`requests/${id}/lock`),
469469
method: 'DELETE',
470-
timeout: this.timeoutMillis,
470+
timeout: Math.min(SMALL_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
471471
params: this._params({
472472
forefront: options.forefront,
473473
clientKey: this.clientKey,
@@ -492,7 +492,7 @@ export class RequestQueueClient extends ResourceClient {
492492
const response = await this.httpClient.call({
493493
url: this._url('requests'),
494494
method: 'GET',
495-
timeout: this.timeoutMillis,
495+
timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
496496
params: this._params({
497497
limit: options.limit,
498498
exclusiveStartId: options.exclusiveStartId,
@@ -510,7 +510,7 @@ export class RequestQueueClient extends ResourceClient {
510510
const response = await this.httpClient.call({
511511
url: this._url('requests/unlock'),
512512
method: 'POST',
513-
timeout: this.timeoutMillis,
513+
timeout: Math.min(MEDIUM_TIMEOUT_MILLIS, this.timeoutMillis ?? Infinity),
514514
params: this._params({
515515
clientKey: this.clientKey,
516516
}),

0 commit comments

Comments
 (0)