Reduce 429 error responses when checking links (#4436)

- set concurrency to 1
- retry 429 responses with retry-after header
- set bearer token for GitHub links

Author: philippthun

.github/workflows/docs_test.yml

@@ -34,6 +34,8 @@ jobs:
         working-directory: docs/v2
         run: ./render.sh
       - name: Run docs tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           bundle config set --local without 'development'
           bundle install

docs/v3/gulpfile.js

@@ -4,8 +4,25 @@ import gulp from 'gulp';
 import express from 'express';
 import { glob } from 'glob';
 import { LinkChecker } from 'linkinator';
-
 import * as cheerio from 'cheerio';
+import { instance as gaxios } from 'gaxios';
+
+const githubToken = process.env.GITHUB_TOKEN;
+console.log('GITHUB_TOKEN is', githubToken ? 'SET' : 'NOT SET');
+
+// Add an interceptor to gaxios to include the GitHub token in the headers for requests to GitHub
+gaxios.interceptors.request.add({
+  resolved: (config) => {
+    if (githubToken && config.url?.includes('github.com')) {
+      config.headers = {
+        ...(config.headers || {}),
+        authorization: `Bearer ${githubToken}`,
+      };
+    }
+    return config;
+  },
+  rejected: (error) => Promise.reject(error),
+});
 
 function displayErrors(err, stdout, stderr) {
   if (err) {
@@ -91,6 +108,8 @@ async function checkPathAndExit(path, options, done) {
     recurse: options.recurse,
     silent: options.silent,
     markdown: options.markdown,
+    concurrency: 1,
+    retry: true,
   };
 
   try {