Reduce 429 error responses when checking links

- set concurrency to 1
- retry 429 responses with retry-after header
- set bearer token for GitHub links

Author: philippthun

.github/workflows/docs_test.yml

@@ -34,6 +34,8 @@ jobs:
         working-directory: docs/v2
         run: ./render.sh
       - name: Run docs tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           bundle config set --local without 'development'
           bundle install

docs/v3/gulpfile.js

@@ -4,6 +4,7 @@ import gulp from 'gulp';
 import express from 'express';
 import { glob } from 'glob';
 import { LinkChecker } from 'linkinator';
+import { instance as gaxios } from 'gaxios';
 
 import * as cheerio from 'cheerio';
 
@@ -85,17 +86,68 @@ async function checkPathAndExit(path, options, done) {
 
   const url = 'http://localhost:8001/';
 
+  const githubToken = process.env.GITHUB_TOKEN;
+
   const config = {
     path: url,
     linksToSkip: options.linksToSkip || [],
     recurse: options.recurse,
     silent: options.silent,
     markdown: options.markdown,
+    concurrency: 1,
+    retry: true,
+    beforeRequest: (url, options) => {
+      // add the bearer token for GitHub links
+      if (githubToken && url.host.endsWith('github.com')) {
+        options.headers = {
+          ...(options.headers || {}),
+          authorization: `Bearer ${githubToken}`,
+        };
+      }
+      return options;
+    },
   };
 
   try {
+    gaxios.interceptors.response.add({
+      // onFulfilled
+      resolved: (response) => {
+        if (response.config.url?.includes('github.com')) {
+          console.log(
+              `→ [${response.status}] ${response.config.url}\n`,
+              response.headers
+          );
+        }
+        return response;
+      },
+      // onRejected (catches 4xx/5xx too)
+      rejected: (error) => {
+        const resp = error.response;
+        if (resp?.status === 429 && resp.config.url.includes('github.com')) {
+          console.error(
+              `→ 429 on ${resp.config.url}\n`,
+              'Request headers:', resp.config.headers,
+              '\nResponse headers:', resp.headers
+          );
+        }
+        return Promise.reject(error);
+      },
+    });
+
     const checker = new LinkChecker();
 
+    // Fires after *each* link is checked
+    checker.on('link', (result) => {
+      if (result.state === 'BROKEN') {
+        // failureDetails is an array; each entry has `.headers`
+        const detail = result.failureDetails[0];
+        console.log(
+            `[${result.status}] ${result.url}`,
+            '\n→ headers:', detail.headers
+        );
+      }
+    });
+
     if (path === '../v2') {
       const htmlFiles = await glob(path + '/**/*.html');
       let allResults = { links: [] };