diff --git a/docs/classes/google-spreadsheet.md b/docs/classes/google-spreadsheet.md index dcac8cb..8dbf054 100644 --- a/docs/classes/google-spreadsheet.md +++ b/docs/classes/google-spreadsheet.md @@ -9,7 +9,7 @@ _Class Reference_ ## Initialization ### Existing documents -#### `new GoogleSpreadsheet(id, auth)` :id=fn-newGoogleSpreadsheet +#### `new GoogleSpreadsheet(id, auth, rateLimitedRetryConfig)` :id=fn-newGoogleSpreadsheet > Work with an existing document > You'll need the document ID, which you can find in your browser's URL when you navigate to the document.
@@ -19,7 +19,7 @@ Param|Type|Required|Description ---|---|---|--- `spreadsheetId` | String | ✅ | Document ID `auth` | `GoogleAuth` \|
`JWT` \|
`OAuth2Client` \|
`{ apiKey: string }` \|
`{ token: string }` | ✅ | Authentication to use
See [Authentication](guides/authentication) for more info - +`rateLimitedRetryConfig` | `{`
`maxRetries: number,`
`retryStrategy: (retryCount: number) => number`
`}` | ❎ | Configure handling for rate limited responses ### Creating a new document diff --git a/package.json b/package.json index 4c9431f..a29908a 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "eslint-plugin-import": "^2.27.5", "eslint-plugin-no-floating-promise": "^1.0.2", "google-auth-library": "^9.14.0", + "nock": "^13.5.5", "release-it": "^15.11.0", "ts-node": "^10.9.1", "typescript": "^5.5.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index be783b7..2a22197 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -69,6 +69,9 @@ importers: google-auth-library: specifier: ^9.14.0 version: 9.14.0 + nock: + specifier: ^13.5.5 + version: 13.5.5 release-it: specifier: ^15.11.0 version: 15.11.0 @@ -2732,6 +2735,9 @@ packages: json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -3052,6 +3058,10 @@ packages: resolution: {integrity: sha512-NHDDGYudnvRutt/VhKFlX26IotXe1w0cmkDm6JGquh5bz/bDTw0LufSmH/GxTjEdpHEO+bVKFTwdrcGa/9XlKQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + nock@13.5.5: + resolution: {integrity: sha512-XKYnqUrCwXC8DGG1xX4YH5yNIrlh9c065uaMZZHUoeUUINTOyt+x/G+ezYk0Ft6ExSREVIs+qBJDK503viTfFA==} + engines: {node: '>= 10.13'} + node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -3348,6 +3358,10 @@ packages: resolution: {integrity: sha512-22wJUOD3zswWFqgwjNHa1965LvqTX87WPu/lreY2KSd7SVcERfuZ4GfUaOnJNnvtoIv2yXT/W00YIGMetXtFXg==} engines: {node: '>= 0.4'} + propagate@2.0.1: + resolution: {integrity: sha512-vGrhOavPSTz4QVNuBNdcNXePNdNMaO1xj9yBeH1ScQPjk/rhg9sSlCXPhMkFuaNNW/syTvYqsnbIJxMBfRbbag==} + engines: {node: '>= 8'} + proto-list@1.2.4: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} @@ -4238,7 +4252,7 @@ snapshots: '@babel/traverse': 7.22.1 '@babel/types': 7.22.0 convert-source-map: 1.9.0 - debug: 4.3.4 + debug: 4.3.6 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.0 @@ -4339,7 +4353,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.18.6 '@babel/parser': 7.22.0 '@babel/types': 7.22.0 - debug: 4.3.4 + debug: 4.3.6 globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -5213,7 +5227,7 @@ snapshots: agent-base@7.1.0: dependencies: - debug: 4.3.4 + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -6551,7 +6565,7 @@ snapshots: dependencies: basic-ftp: 5.0.3 data-uri-to-buffer: 5.0.1 - debug: 4.3.4 + debug: 4.3.6 fs-extra: 8.1.0 transitivePeerDependencies: - supports-color @@ -6769,7 +6783,7 @@ snapshots: http-proxy-agent@7.0.0: dependencies: agent-base: 7.1.0 - debug: 4.3.4 + debug: 4.3.6 transitivePeerDependencies: - supports-color @@ -7090,6 +7104,8 @@ snapshots: json-stable-stringify-without-jsonify@1.0.1: {} + json-stringify-safe@5.0.1: {} + json5@1.0.2: dependencies: minimist: 1.2.8 @@ -7366,6 +7382,14 @@ snapshots: dependencies: type-fest: 2.19.0 + nock@13.5.5: + dependencies: + debug: 4.3.6 + json-stringify-safe: 5.0.1 + propagate: 2.0.1 + transitivePeerDependencies: + - supports-color + node-domexception@1.0.0: {} node-fetch@2.6.11: @@ -7546,7 +7570,7 @@ snapshots: pac-proxy-agent@6.0.3: dependencies: agent-base: 7.1.0 - debug: 4.3.4 + debug: 4.3.6 get-uri: 6.0.1 http-proxy-agent: 7.0.0 https-proxy-agent: 7.0.2 @@ -7661,6 +7685,8 @@ snapshots: get-intrinsic: 1.2.1 iterate-value: 1.0.2 + propagate@2.0.1: {} + proto-list@1.2.4: {} protocols@2.0.1: {} @@ -7992,7 +8018,7 @@ snapshots: socks-proxy-agent@8.0.1: dependencies: agent-base: 7.1.0 - debug: 4.3.4 + debug: 4.3.6 socks: 2.7.1 transitivePeerDependencies: - supports-color diff --git a/src/lib/GoogleSpreadsheet.ts b/src/lib/GoogleSpreadsheet.ts index 656d3f3..0c26318 100644 --- a/src/lib/GoogleSpreadsheet.ts +++ b/src/lib/GoogleSpreadsheet.ts @@ -28,8 +28,29 @@ const EXPORT_CONFIG: Record = { }; type ExportFileTypes = keyof typeof EXPORT_CONFIG; +type RateLimitedRetryConfig = { + /** + * The maximum number of times to retry the request. 0 means no retries. + */ + maxRetries: number, + /** + * A function that takes the current retry count and returns the number of milliseconds to wait before the next retry. + * @see https://developers.google.com/sheets/api/limits#example-algorithm + */ + retryStrategy: (retryCount: number) => number, +}; - +/** + * Default rate limited retry configuration based on the Google Sheets API example algorithm. + * The wait time is min(((2^n)+random_number_milliseconds), maximum_backoff), with n incremented by 1 for each iteration (request). + * random_number_milliseconds is a random number of milliseconds less than or equal to 1,000. + * maximum_backoff is typically 32 or 64 seconds. The appropriate value depends on the use case. + * @see https://developers.google.com/sheets/api/limits#example-algorithm + */ +const DEFAULT_RATE_LIMITED_RETRY_CONFIG: RateLimitedRetryConfig = { + maxRetries: 3, + retryStrategy: (retryCount) => Math.min(2 ** retryCount + Math.random() * 100, 32 * 1000), +}; function getAuthMode(auth: GoogleApiAuth) { if ('getRequestHeaders' in auth) return AUTH_MODES.GOOGLE_AUTH_CLIENT; @@ -81,6 +102,7 @@ export class GoogleSpreadsheet { private _rawProperties = null as SpreadsheetProperties | null; private _spreadsheetUrl = null as string | null; private _deleted = false; + private _rateLimitedRetryConfig: RateLimitedRetryConfig | undefined; /** * Sheets API [axios](https://axios-http.com) instance @@ -108,7 +130,10 @@ export class GoogleSpreadsheet { /** id of google spreadsheet doc */ spreadsheetId: SpreadsheetId, /** authentication to use with Google Sheets API */ - auth: GoogleApiAuth + auth: GoogleApiAuth, + options?: { + retryOnRateLimit?: true | RateLimitedRetryConfig + } ) { this.spreadsheetId = spreadsheetId; this.auth = auth; @@ -133,15 +158,19 @@ export class GoogleSpreadsheet { this.sheetsApi.interceptors.request.use(this._setAxiosRequestAuth.bind(this)); this.sheetsApi.interceptors.response.use( this._handleAxiosResponse.bind(this), - this._handleAxiosErrors.bind(this) + this._handleAxiosErrors(this.sheetsApi).bind(this) ); this.driveApi.interceptors.request.use(this._setAxiosRequestAuth.bind(this)); this.driveApi.interceptors.response.use( this._handleAxiosResponse.bind(this), - this._handleAxiosErrors.bind(this) + this._handleAxiosErrors(this.driveApi).bind(this) ); - } + if (options?.retryOnRateLimit) { + const retryOptions = options.retryOnRateLimit; + this._rateLimitedRetryConfig = retryOptions === true ? DEFAULT_RATE_LIMITED_RETRY_CONFIG : retryOptions; + } + } // AUTH RELATED FUNCTIONS //////////////////////////////////////////////////////////////////////// @@ -160,25 +189,48 @@ export class GoogleSpreadsheet { /** @internal */ async _handleAxiosResponse(response: AxiosResponse) { return response; } /** @internal */ - async _handleAxiosErrors(error: AxiosError) { - // console.log(error); - const errorData = error.response?.data as any; + _handleAxiosErrors(axiosInstance: AxiosInstance) { + return async (error: AxiosError) => { + const responseStatusCode = _.get(error, 'response.status'); + + // Handle rate limited responses by retrying based on the rate limited retry config + if (responseStatusCode === 429) { + const config = error.config as InternalAxiosRequestConfig & { retryCount?: number }; + const retryCount = config?.retryCount ?? 0; + + const rateLimitedRetryConfig = this._rateLimitedRetryConfig; + if (rateLimitedRetryConfig && retryCount < rateLimitedRetryConfig.maxRetries) { + config.retryCount = retryCount + 1; + const backoff = rateLimitedRetryConfig.retryStrategy(retryCount); + return new Promise((resolve) => { + setTimeout( + () => { + resolve(axiosInstance(config)); + }, + backoff + ); + }); + } + } + // console.log(error); + const errorData = error.response?.data as any; - if (errorData) { + if (errorData) { // usually the error has a code and message, but occasionally not - if (!errorData.error) throw error; + if (!errorData.error) throw error; - const { code, message } = errorData.error; - error.message = `Google API error - [${code}] ${message}`; - throw error; - } + const { code, message } = errorData.error; + error.message = `Google API error - [${code}] ${message}`; + throw error; + } - if (_.get(error, 'response.status') === 403) { - if ('apiKey' in this.auth) { - throw new Error('Sheet is private. Use authentication or make public. (see https://github.com/theoephraim/node-google-spreadsheet#a-note-on-authentication for details)'); + if (responseStatusCode === 403) { + if ('apiKey' in this.auth) { + throw new Error('Sheet is private. Use authentication or make public. (see https://github.com/theoephraim/node-google-spreadsheet#a-note-on-authentication for details)'); + } } - } - throw error; + throw error; + }; } /** @internal */ diff --git a/src/test/rateLimitedHandling.test.ts b/src/test/rateLimitedHandling.test.ts new file mode 100644 index 0000000..8b30416 --- /dev/null +++ b/src/test/rateLimitedHandling.test.ts @@ -0,0 +1,110 @@ +import { + describe, expect, it, +} from 'vitest'; +import nock from 'nock'; +import axios from 'axios'; +import { GoogleSpreadsheet } from '../lib/GoogleSpreadsheet'; + +axios.defaults.adapter = 'http'; + +const SPREADSHEET_ID = '123456'; +const SPREADSHEET_URI_PART = `/${SPREADSHEET_ID}/`; +const SHEETS_API = 'https://sheets.googleapis.com/v4/spreadsheets'; +const MOCKED_SHEET = { + spreadsheetUrl: `https://docs.google.com/spreadsheets/d/${SPREADSHEET_ID}/`, + properties: [], +}; + +const sheetRetryTests = (doc: GoogleSpreadsheet) => { + it('does not affect non-rate-limited requests', async () => { + const scope = nock(SHEETS_API) + .get(SPREADSHEET_URI_PART) + .reply(200, MOCKED_SHEET); + await doc.loadInfo(); + scope.done(); + }); + + it('retries the max amount and then throws if still rate limited', async () => { + const scope = nock(SHEETS_API) + .get(SPREADSHEET_URI_PART) + .times(4) + .reply(429, {}); + await expect(doc.loadInfo()).rejects.toHaveProperty('message', 'Request failed with status code 429'); + scope.done(); + }); + + it('retries the max amount and then succeeds if no longer rate limited', async () => { + const scope = nock(SHEETS_API) + .get(SPREADSHEET_URI_PART) + .times(3) + .reply(429, {}) + .get(SPREADSHEET_URI_PART) + .reply(200, MOCKED_SHEET); + await doc.loadInfo(); + scope.done(); + }); +}; + +describe('Rate limited handling configured with custom implementation', async () => { + const doc = new GoogleSpreadsheet( + SPREADSHEET_ID, + { + getRequestHeaders: async () => ({ + Authorization: 'Bearer fake-access-token', + }), + }, + { + retryOnRateLimit: { + maxRetries: 3, + retryStrategy: () => 1000, + }, + } + ); + + sheetRetryTests(doc); +}); + +describe('Rate limited handling configured with default implementation', async () => { + const doc = new GoogleSpreadsheet( + SPREADSHEET_ID, + { + getRequestHeaders: async () => ({ + Authorization: 'Bearer fake-access-token', + }), + }, + { + retryOnRateLimit: true, + } + ); + + sheetRetryTests(doc); +}); + +describe('Rate limited handling not enabled', async () => { + const doc = new GoogleSpreadsheet( + SPREADSHEET_ID, + { + getRequestHeaders: async () => ({ + Authorization: 'Bearer fake-access-token', + }), + }, + { + } + ); + + it('does not affect non-rate-limited requests', async () => { + const scope = nock(SHEETS_API) + .get(SPREADSHEET_URI_PART) + .reply(200, MOCKED_SHEET); + await doc.loadInfo(); + scope.done(); + }); + + it('throws if rate limited', async () => { + const scope = nock(SHEETS_API) + .get(SPREADSHEET_URI_PART) + .reply(429, {}); + await expect(doc.loadInfo()).rejects.toHaveProperty('message', 'Request failed with status code 429'); + scope.done(); + }); +});