Reduce 429 error responses when checking links

- set concurrency to 1
- retry 429 responses with retry-after header
- set bearer token for GitHub links

Author: philippthun

.github/workflows/docs_test.yml

@@ -34,6 +34,8 @@ jobs:
         working-directory: docs/v2
         run: ./render.sh
       - name: Run docs tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           bundle config set --local without 'development'
           bundle install

docs/v3/gulpfile.js

@@ -85,12 +85,26 @@ async function checkPathAndExit(path, options, done) {
 
   const url = 'http://localhost:8001/';
 
+  const githubToken = process.env.GITHUB_TOKEN;
+
   const config = {
     path: url,
     linksToSkip: options.linksToSkip || [],
     recurse: options.recurse,
     silent: options.silent,
     markdown: options.markdown,
+    concurrency: 1,
+    retry: true,
+    beforeRequest: (url, options) => {
+      // add the bearer token for GitHub links
+      if (githubToken && url.host.endsWith('github.com')) {
+        options.headers = {
+          ...(options.headers || {}),
+          authorization: `Bearer ${githubToken}`,
+        };
+      }
+      return options;
+    },
   };
 
   try {
@@ -108,6 +122,11 @@ async function checkPathAndExit(path, options, done) {
       displayResults(allResults);
     } else {
       const results = await checker.check(config);
+
+      const userCall = results.find(r => r.url.includes('github.com'));
+      console.log('Status:', userCall.status);
+      console.log('Headers:', userCall.response.headers);
+
       displayResults(results);
     }