fix(scripts): add retry rate limiting

unmerged code from theoephraim/node-google-spreadsheet#525

Author: steveoh

scripts/package-lock.json

@@ -5,6 +5,7 @@
   },
   "dependencies": {
     "@octokit/rest": "^21.0.1",
+    "axios-retry": "^4.5.0",
     "change-case": "^5.4.4",
     "fast-xml-parser": "^4.4.1",
     "glob": "^11.0.0",

scripts/package.json

@@ -1,3 +1,4 @@
+import axiosRetry from 'axios-retry';
 import { GoogleAuth, auth } from 'google-auth-library';
 import { GoogleSpreadsheet } from 'google-spreadsheet';
 import jsonToMarkdown from 'json-to-markdown-table';
@@ -7,6 +8,17 @@ import * as tsImport from 'ts-import';
 import { v4 as uuid } from 'uuid';
 import { slugify, validateOpenDataUrl, validateOpenSgidTableName, validateUrl } from './utilities.mjs';
 
+function retry(client) {
+  axiosRetry(client, {
+    retries: 7,
+    retryDelay: (retryCount) => {
+      const randomNumberMS = _.random(1000, 8000);
+      return Math.min(4 ** retryCount + randomNumberMS, maximumBackoff);
+    },
+    retryCondition: (error) => error.response.status === 429,
+  });
+}
+
 const downloadMetadata = await tsImport.load('../src/data/downloadMetadata.ts');
 
 const spreadsheetId = '11ASS7LnxgpnD0jN4utzklREgMf1pcvYjcXcIcESHweQ';
@@ -26,6 +38,9 @@ if (process.env.GITHUB_ACTIONS) {
 
 console.log('loading spreadsheet');
 const spreadsheet = new GoogleSpreadsheet(spreadsheetId, client);
+retry(spreadsheet.sheetsApi);
+retry(spreadsheet.driveApi);
+
 await spreadsheet.loadInfo();
 const worksheet = spreadsheet.sheetsByTitle['SGID Index'];