Reduce 429 error responses when checking links

- set concurrency to 1
- retry 429 responses with retry-after header
- set bearer token for GitHub links

Author: philippthun

.github/workflows/docs_test.yml

@@ -34,6 +34,8 @@ jobs:
         working-directory: docs/v2
         run: ./render.sh
       - name: Run docs tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           bundle config set --local without 'development'
           bundle install

docs/v3/gulpfile.js

@@ -4,6 +4,7 @@ import gulp from 'gulp';
 import express from 'express';
 import { glob } from 'glob';
 import { LinkChecker } from 'linkinator';
+import { instance as gaxios } from 'gaxios';
 
 import * as cheerio from 'cheerio';
 
@@ -85,17 +86,65 @@ async function checkPathAndExit(path, options, done) {
 
   const url = 'http://localhost:8001/';
 
+  const githubToken = process.env.GITHUB_TOKEN;
+
   const config = {
     path: url,
     linksToSkip: options.linksToSkip || [],
     recurse: options.recurse,
     silent: options.silent,
     markdown: options.markdown,
+    concurrency: 1,
+    retry: true,
+    beforeRequest: (url, options) => {
+      // add the bearer token for GitHub links
+      if (githubToken && url.host.endsWith('github.com')) {
+        options.headers = {
+          ...(options.headers || {}),
+          authorization: `Bearer ${githubToken}`,
+        };
+      }
+      return options;
+    },
   };
 
   try {
+    const token = process.env.GITHUB_TOKEN;
+    console.log('GITHUB_TOKEN is', token ? 'SET' : 'NOT SET');
+
+    gaxios.interceptors.request.add({
+      resolved: (config) => {
+        if (config.url?.includes('github.com')) {
+          console.log('→ Request to', config.url);
+          console.log('   Request headers:', config.headers);
+
+          if (token) {
+            config.headers = {
+              ...(config.headers || {}),
+              authorization: `Bearer ${token}`,
+            };
+            console.log('   Headers after injection: ', config.headers);
+          }
+        }
+        return config;
+      },
+      rejected: (error) => Promise.reject(error),
+    });
+
     const checker = new LinkChecker();
 
+    // Fires after *each* link is checked
+    checker.on('link', (result) => {
+      if (result.state === 'BROKEN') {
+        // failureDetails is an array; each entry has `.headers`
+        const detail = result.failureDetails[0];
+        console.log(
+            `[${result.status}] ${result.url}`,
+            '\n→ headers:', detail.headers
+        );
+      }
+    });
+
     if (path === '../v2') {
       const htmlFiles = await glob(path + '/**/*.html');
       let allResults = { links: [] };