fix: Use appropriate timeouts

src/apify_client.ts

@@ -33,6 +33,8 @@ import { WebhookDispatchClient } from './resource_clients/webhook_dispatch';
 import { WebhookDispatchCollectionClient } from './resource_clients/webhook_dispatch_collection';
 import { Statistics } from './statistics';
 
+const DEFAULT_TIMEOUT_SECS = 360;
+
 /**
  * ApifyClient is the official library to access [Apify API](https://docs.apify.com/api/v2) from your
  * JavaScript applications. It runs both in Node.js and browser.
@@ -67,7 +69,7 @@ export class ApifyClient {
             maxRetries = 8,
             minDelayBetweenRetriesMillis = 500,
             requestInterceptors = [],
-            timeoutSecs = 360,
+            timeoutSecs = DEFAULT_TIMEOUT_SECS,
             token,
         } = options;
 

src/base/resource_client.ts

@@ -18,11 +18,12 @@ const MAX_WAIT_FOR_FINISH = 999999;
  * @private
  */
 export class ResourceClient extends ApiClient {
-    protected async _get<T, R>(options: T = {} as T): Promise<R | undefined> {
+    protected async _get<T, R>(options: T = {} as T, timeoutSecs?: number): Promise<R | undefined> {
         const requestOpts: ApifyRequestConfig = {
             url: this._url(),
             method: 'GET',
             params: this._params(options),
+            timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : undefined,
         };
         try {
             const response = await this.httpClient.call(requestOpts);
@@ -34,22 +35,24 @@ export class ResourceClient extends ApiClient {
         return undefined;
     }
 
-    protected async _update<T, R>(newFields: T): Promise<R> {
+    protected async _update<T, R>(newFields: T, timeoutSecs?: number): Promise<R> {
         const response = await this.httpClient.call({
             url: this._url(),
             method: 'PUT',
             params: this._params(),
             data: newFields,
+            timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : undefined,
         });
         return parseDateFields(pluckData(response.data)) as R;
     }
 
-    protected async _delete(): Promise<void> {
+    protected async _delete(timeoutSecs?: number): Promise<void> {
         try {
             await this.httpClient.call({
                 url: this._url(),
                 method: 'DELETE',
                 params: this._params(),
+                timeout: timeoutSecs !== undefined ? timeoutSecs * 1000 : undefined,
             });
         } catch (err) {
             catchNotFoundOrThrow(err as ApifyApiError);

src/http_client.ts

@@ -152,13 +152,21 @@ export class HttpClient {
             this.stats.requests++;
             let response: ApifyResponse;
             const requestIsStream = isStream(config.data);
+
             try {
                 if (requestIsStream) {
                     // Handling redirects is not possible without buffering - part of the stream has already been sent and can't be recovered
                     // when server sends the redirect. Therefore we need to override this in Axios config to prevent it from buffering the body.
                     // see also axios/axios#1045
                     config = { ...config, maxRedirects: 0 };
                 }
+
+                // Increase timeout with each attempt. Max timeout is bounded by the client timeout.
+                config.timeout = Math.min(
+                    this.timeoutMillis,
+                    config.timeout ?? this.timeoutMillis * 2 ** (attempt - 1),
+                );
+
                 response = await this.axios.request(config);
                 if (this._isStatusOk(response.status)) return response;
             } catch (err) {

src/resource_clients/dataset.ts

@@ -9,6 +9,10 @@ import type { ApifyRequestConfig, ApifyResponse } from '../http_client';
 import type { PaginatedList } from '../utils';
 import { cast, catchNotFoundOrThrow, pluckData } from '../utils';
 
+const SMALL_TIMEOUT_SECS = 5; // For fast and common actions. Suitable for idempotent actions.
+const MEDIUM_TIMEOUT_SECS = 30; // For actions that may take longer.
+const DEFAULT_TIMEOUT_SECS = 360; // 6 minutes
+
 export class DatasetClient<
     Data extends Record<string | number, any> = Record<string | number, unknown>,
 > extends ResourceClient {
@@ -26,7 +30,7 @@ export class DatasetClient<
      * https://docs.apify.com/api/v2#/reference/datasets/dataset/get-dataset
      */
     async get(): Promise<Dataset | undefined> {
-        return this._get();
+        return this._get({}, SMALL_TIMEOUT_SECS);
     }
 
     /**
@@ -35,14 +39,14 @@ export class DatasetClient<
     async update(newFields: DatasetClientUpdateOptions): Promise<Dataset> {
         ow(newFields, ow.object);
 
-        return this._update(newFields);
+        return this._update(newFields, SMALL_TIMEOUT_SECS);
     }
 
     /**
      * https://docs.apify.com/api/v2#/reference/datasets/dataset/delete-dataset
      */
     async delete(): Promise<void> {
-        return this._delete();
+        return this._delete(SMALL_TIMEOUT_SECS);
     }
 
     /**
@@ -70,6 +74,7 @@ export class DatasetClient<
             url: this._url('items'),
             method: 'GET',
             params: this._params(options),
+            timeout: DEFAULT_TIMEOUT_SECS * 1000,
         });
 
         return this._createPaginationList(response, options.desc ?? false);
@@ -113,6 +118,7 @@ export class DatasetClient<
                 ...options,
             }),
             forceBuffer: true,
+            timeout: DEFAULT_TIMEOUT_SECS * 1000,
         });
 
         return cast(data);
@@ -133,6 +139,7 @@ export class DatasetClient<
             data: items,
             params: this._params(),
             doNotRetryTimeouts: true, // see timeout handling in http-client
+            timeout: MEDIUM_TIMEOUT_SECS * 1000,
         });
     }
 
@@ -144,6 +151,7 @@ export class DatasetClient<
             url: this._url('statistics'),
             method: 'GET',
             params: this._params(),
+            timeout: SMALL_TIMEOUT_SECS * 1000,
         };
         try {
             const response = await this.httpClient.call(requestOpts);

src/resource_clients/key_value_store.ts

@@ -12,6 +12,10 @@ import { ResourceClient } from '../base/resource_client';
 import type { ApifyRequestConfig } from '../http_client';
 import { cast, catchNotFoundOrThrow, isBuffer, isNode, isStream, parseDateFields, pluckData } from '../utils';
 
+const SMALL_TIMEOUT_SECS = 5; // For fast and common actions. Suitable for idempotent actions.
+const MEDIUM_TIMEOUT_SECS = 30; // For actions that may take longer.
+const DEFAULT_TIMEOUT_SECS = 360; // 6 minutes
+
 export class KeyValueStoreClient extends ResourceClient {
     /**
      * @hidden
@@ -27,7 +31,7 @@ export class KeyValueStoreClient extends ResourceClient {
      * https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/get-store
      */
     async get(): Promise<KeyValueStore | undefined> {
-        return this._get();
+        return this._get({}, SMALL_TIMEOUT_SECS);
     }
 
     /**
@@ -36,14 +40,14 @@ export class KeyValueStoreClient extends ResourceClient {
     async update(newFields: KeyValueClientUpdateOptions): Promise<KeyValueStore> {
         ow(newFields, ow.object);
 
-        return this._update(newFields);
+        return this._update(newFields, DEFAULT_TIMEOUT_SECS);
     }
 
     /**
      * https://docs.apify.com/api/v2#/reference/key-value-stores/store-object/delete-store
      */
     async delete(): Promise<void> {
-        return this._delete();
+        return this._delete(SMALL_TIMEOUT_SECS);
     }
 
     /**
@@ -64,6 +68,7 @@ export class KeyValueStoreClient extends ResourceClient {
             url: this._url('keys'),
             method: 'GET',
             params: this._params(options),
+            timeout: MEDIUM_TIMEOUT_SECS * 1000,
         });
 
         return cast(parseDateFields(pluckData(response.data)));
@@ -138,6 +143,7 @@ export class KeyValueStoreClient extends ResourceClient {
             url: this._url(`records/${key}`),
             method: 'GET',
             params: this._params(),
+            timeout: DEFAULT_TIMEOUT_SECS * 1000,
         };
 
         if (options.buffer) requestOpts.forceBuffer = true;
@@ -181,14 +187,14 @@ export class KeyValueStoreClient extends ResourceClient {
         ow(
             options,
             ow.object.exactShape({
-                timeoutSecs: ow.optional.number,
+                timeoutMillis: ow.optional.number,
                 doNotRetryTimeouts: ow.optional.boolean,
             }),
         );
 
         const { key } = record;
         let { value, contentType } = record;
-        const { timeoutSecs, doNotRetryTimeouts } = options;
+        const { timeoutMillis, doNotRetryTimeouts } = options;
 
         const isValueStreamOrBuffer = isStream(value) || isBuffer(value);
         // To allow saving Objects to JSON without providing content type
@@ -215,12 +221,9 @@ export class KeyValueStoreClient extends ResourceClient {
             data: value,
             headers: contentType ? { 'content-type': contentType } : undefined,
             doNotRetryTimeouts,
+            timeout: timeoutMillis ?? DEFAULT_TIMEOUT_SECS * 1000,
         };
 
-        if (timeoutSecs != null) {
-            uploadOpts.timeout = timeoutSecs * 1000;
-        }
-
         await this.httpClient.call(uploadOpts);
     }
 
@@ -234,6 +237,7 @@ export class KeyValueStoreClient extends ResourceClient {
             url: this._url(`records/${key}`),
             method: 'DELETE',
             params: this._params(),
+            timeout: SMALL_TIMEOUT_SECS * 1000,
         });
     }
 }
@@ -299,7 +303,7 @@ export interface KeyValueStoreRecord<T> {
 }
 
 export interface KeyValueStoreRecordOptions {
-    timeoutSecs?: number;
+    timeoutMillis?: number;
     doNotRetryTimeouts?: boolean;
 }